From d418d9bdfb6f8d163dc779021f1d37981e23e1ed Mon Sep 17 00:00:00 2001 From: Exelo Date: Fri, 30 Oct 2026 10:33:37 +0100 Subject: [PATCH 1/2] feat: add cache system with OPFS --- .github/workflows/e2e.yml | 2 +- .github/workflows/tests.yml | 6 +- CLAUDE.md | 1 + apps/editor/e2e/cache.spec.ts | 78 ++++ apps/editor/e2e/project.spec.ts | 1 + apps/editor/playwright.config.ts | 27 +- apps/editor/src/lib/editor/create-editor.ts | 2 + .../src/lib/project/follow-project-caches.ts | 60 +++ apps/editor/src/routes/(editor)/+page.svelte | 3 + docs/docs/editor/settings.mdx | 8 + packages/code/src/service/code-service.ts | 8 + packages/project/package.json | 4 +- packages/project/src/cache/cache-db.ts | 81 ++++ packages/project/src/cache/cache.type.ts | 20 +- packages/project/src/cache/index.ts | 3 +- .../src/cache/indexed-db.tree-cache.ts | 43 +- .../project/src/cache/memory.content-cache.ts | 24 +- packages/project/src/cache/opfs-write.ts | 87 +++++ .../project/src/cache/opfs.content-cache.ts | 111 +++--- packages/project/src/cache/project-caches.ts | 369 ++++++++++++++++++ packages/project/src/cache/sha1.ts | 5 + packages/project/src/fs/project-fs.ts | 161 +++++++- .../project/src/service/client-project.ts | 17 +- .../project/src/service/project-service.ts | 9 +- .../test/cache/content-cache.browser.spec.ts | 65 +++ .../test/cache/project-caches.browser.spec.ts | 133 +++++++ packages/project/test/fs/project-fs.spec.ts | 71 +++- packages/project/vitest.config.ts | 44 +++ packages/protocol/src/fs/fs.contract.ts | 11 +- packages/protocol/src/fs/fs.schema.ts | 23 ++ packages/sdk/src/index.ts | 4 + packages/sdk/test/index.spec.ts | 1 + packages/server-core/src/fs/file-hashes.ts | 116 ++++++ packages/server-core/src/fs/fs-service.ts | 109 +++++- .../src/project/project-registry.ts | 14 +- .../rpc/implementation/fs.implementation.ts | 2 + .../src/server/create-editor-server.ts | 8 + .../server-core/test/fs/file-hashes.spec.ts | 32 ++ .../server-core/test/fs/fs-service.spec.ts | 29 ++ .../test/server/create-editor-server.spec.ts | 9 +- .../settings/src/definition/core-settings.ts | 20 + .../code-editor/src/typing/type-libraries.ts | 2 + .../dialogs/settings/SettingsDialog.svelte | 2 + .../src/dialogs/settings/StoragePanel.svelte | 152 ++++++++ pnpm-lock.yaml | 6 + 45 files changed, 1800 insertions(+), 183 deletions(-) create mode 100644 apps/editor/e2e/cache.spec.ts create mode 100644 apps/editor/src/lib/project/follow-project-caches.ts create mode 100644 packages/project/src/cache/cache-db.ts create mode 100644 packages/project/src/cache/opfs-write.ts create mode 100644 packages/project/src/cache/project-caches.ts create mode 100644 packages/project/src/cache/sha1.ts create mode 100644 packages/project/test/cache/content-cache.browser.spec.ts create mode 100644 packages/project/test/cache/project-caches.browser.spec.ts create mode 100644 packages/project/vitest.config.ts create mode 100644 packages/server-core/src/fs/file-hashes.ts create mode 100644 packages/server-core/test/fs/file-hashes.spec.ts create mode 100644 plugins/settings-ui/src/dialogs/settings/StoragePanel.svelte diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index de9b7c2c..19cfa59e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -54,7 +54,7 @@ jobs: working-directory: engine - name: Install browsers - run: pnpm --filter @nanoforge-dev/editor exec playwright install --with-deps chromium + run: pnpm --filter @nanoforge-dev/editor exec playwright install --with-deps chromium firefox - name: Run end-to-end tests run: pnpm --filter @nanoforge-dev/editor test:e2e diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 21eba6ee..078ebcdb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,8 +34,10 @@ jobs: - name: Run type checks run: pnpm typecheck - - name: Install browsers for component tests - run: pnpm --filter @nanoforge-dev/editor-ui exec playwright install --with-deps chromium + - name: Install browsers for component and cache tests + run: pnpm --filter @nanoforge-dev/editor-ui exec playwright install --with-deps chromium firefox webkit - name: Run tests run: pnpm test:unit + env: + BROWSERS: chromium,firefox,webkit diff --git a/CLAUDE.md b/CLAUDE.md index e98e5790..f269ec69 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -259,6 +259,7 @@ Everything in our registry has a `nanoforge.manifest.json` and a name `@scope/na - The server worker exits on `disconnect`. - **CLI:** `nf` delegates to the CLI installed in the project (`/node_modules/@nanoforge-dev/cli/dist`), so `NF_CLI_PATH` does not change which build code runs for such a project. - **Console filters and `e2e`:** the Console's group, source and level filters are widget state (saved server-side). A test that changes them puts them back and waits 900 ms. +- **Browser file cache (M8):** contents are kept by sha1 (`FileEntry.hash`, computed by the server's `FileHashes`, keyed by path, size, mtime, ctime and inode) in OPFS `nanoforge-editor/v2//blobs/`, and the cached tree is the path → hash index (`ProjectCaches`, `packages/project/src/cache`). Every OPFS read checks the bytes against their hash; a write is cached only when its hash is the server's. All IndexedDB stores of the cache open through `cache-db.ts` (one version). Several reads go through `fs.readMany` (`notModified` for current copies). Browser tests: `packages/project` `*.browser.spec.ts` (Chromium, Firefox; `BROWSERS=…,webkit`), e2e `cache.spec.ts` on Chromium and Firefox. RPC calls are HTTP POSTs (`/rpc/.`): watch `request`, not WebSocket frames. - **File manager:** - Fast repeated commands race: they run through a serialized queue. - After a drag and drop, focus goes back to the tree (`refocus`), so Ctrl+Z reaches the Files history rather than the layout. diff --git a/apps/editor/e2e/cache.spec.ts b/apps/editor/e2e/cache.spec.ts new file mode 100644 index 00000000..8df9ad84 --- /dev/null +++ b/apps/editor/e2e/cache.spec.ts @@ -0,0 +1,78 @@ +import { expandFolders, openProject, resetLayout } from './helpers'; +import { type Page, expect, test } from './test'; + +/** Files kept in the browser's cache (OPFS), every user together. */ +const keptFiles = (page: Page) => + page.evaluate(async () => { + let count = 0; + try { + const layout = await ( + await navigator.storage.getDirectory() + ) + .getDirectoryHandle('nanoforge-editor') + .then((root) => root.getDirectoryHandle('v2')); + for await (const user of ( + layout as unknown as { values(): AsyncIterable } + ).values()) { + const blobs = await user.getDirectoryHandle('blobs'); + for await (const name of (blobs as unknown as { keys(): AsyncIterable }).keys()) + if (name) count++; + } + } catch {} + return count; + }); + +const openMain = async (page: Page) => { + await expandFolders(page, 'apps', 'client', 'src'); + await page + .getByRole('tree', { name: 'Project files' }) + .getByRole('treeitem', { name: 'main.ts', exact: true }) + .dblclick(); + await expect(page.getByRole('tab', { name: 'main.ts' })).toBeVisible(); +}; + +test('keeps project files in the browser and reads none of them again after a reload', async ({ + page, +}) => { + test.setTimeout(90_000); + /** File reads the page asks the server for (RPC calls are HTTP requests). */ + const reads: string[] = []; + page.on('request', (request) => { + if (request.url().includes('fs.readMany')) reads.push(request.url()); + }); + await openProject(page, 'pong'); + await page.waitForURL(/\/project\//); + await resetLayout(page); + await openMain(page); + await expect.poll(() => keptFiles(page), { timeout: 15_000 }).toBeGreaterThan(5); + /** The code worker and the code editor's type information fill the cache in the background. */ + await page.waitForTimeout(3000); + /** The first visit reads the files from the server: the check below can fail. */ + expect(reads.length).toBeGreaterThan(0); + + reads.length = 0; + await page.reload(); + await expect(page.getByRole('tab', { name: 'main.ts' })).toBeVisible({ timeout: 15_000 }); + await page.getByRole('tab', { name: 'main.ts' }).click(); + await expect(page.locator('.monaco-editor .view-lines')).toContainText('NanoforgeFactory', { + timeout: 15_000, + }); + await page.waitForTimeout(3000); + expect(reads).toEqual([]); + + await page.locator('[data-nf-part="screen"]').click(); + await page.keyboard.press('Control+Alt+s'); + const dialog = page.getByRole('dialog', { name: 'Settings' }); + await dialog + .getByRole('tree', { name: 'Settings categories' }) + .getByRole('treeitem', { name: 'Storage', exact: true }) + .click(); + const kept = dialog.getByRole('list', { name: 'Kept projects' }); + await expect(kept.getByRole('listitem').filter({ hasText: 'pong-network' })).toBeVisible(); + await dialog.getByRole('button', { name: 'Clear the files kept of pong-network' }).click(); + await expect(kept.getByRole('listitem').filter({ hasText: 'pong-network' })).toHaveCount(0); + await page.keyboard.press('Escape'); + await page.getByRole('tab', { name: 'main.ts' }).click({ button: 'middle' }); + await resetLayout(page); + await page.waitForTimeout(900); +}); diff --git a/apps/editor/e2e/project.spec.ts b/apps/editor/e2e/project.spec.ts index ee2389da..c785ee09 100644 --- a/apps/editor/e2e/project.spec.ts +++ b/apps/editor/e2e/project.spec.ts @@ -95,6 +95,7 @@ test('adds a shared library, chooses the apps that use it, renames and removes i .getByRole('checkbox', { name: 'pong-network-client' }) .uncheck(); await expect.poll(() => dependencies(CLIENT)).toEqual({}); + /** The config is written after the package.json. */ await expect.poll(() => read(CLIENT_CONFIG)).toContain('libs: [],'); expect(dependencies(SERVER)).toEqual({ '@pong-network/shared': 'workspace:*' }); await expect(components.getByRole('button', { name: 'C Score', exact: true })).toHaveCount(0); diff --git a/apps/editor/playwright.config.ts b/apps/editor/playwright.config.ts index e5c2e9c7..2c17b603 100644 --- a/apps/editor/playwright.config.ts +++ b/apps/editor/playwright.config.ts @@ -13,7 +13,17 @@ if (!/^\.workspace(-[\w-]+)?$/.test(WORKSPACE)) * fresh workspace holding a copy of pong-network, with the example plugin as a dev plugin. * Set CHROME to use an installed Chromium instead of Playwright's, NANOFORGE_ENGINE to play the * pong-network example of another engine checkout (default: ../engine next to the editor). + * Everything runs on Chromium; the browser cache spec also on Firefox, and on WebKit with + * `BROWSERS=…,webkit` where it starts (it needs system libraries some distributions lack). */ +const CROSS_BROWSER = 'cache.spec.ts'; +const others = (process.env.BROWSERS ?? 'chromium,firefox') + .split(',') + .map((browser) => browser.trim()) + .filter( + (browser): browser is 'firefox' | 'webkit' => browser === 'firefox' || browser === 'webkit', + ); + export default defineConfig({ testDir: 'e2e', testMatch: process.env.BENCH ? '*.bench.ts' : '*.spec.ts', @@ -24,9 +34,24 @@ export default defineConfig({ use: { baseURL: `http://127.0.0.1:${PORT}`, viewport: { width: 1440, height: 860 }, - launchOptions: process.env.CHROME ? { executablePath: process.env.CHROME } : {}, trace: 'retain-on-failure', }, + projects: [ + { + name: 'chromium', + use: { + browserName: 'chromium', + launchOptions: process.env.CHROME ? { executablePath: process.env.CHROME } : {}, + }, + }, + ...(process.env.BENCH + ? [] + : others.map((browser) => ({ + name: browser, + testMatch: CROSS_BROWSER, + use: { browserName: browser }, + }))), + ], webServer: { command: 'node e2e/prepare-workspace.mjs && bun ./dist/index.js', url: `http://127.0.0.1:${PORT}/healthz`, diff --git a/apps/editor/src/lib/editor/create-editor.ts b/apps/editor/src/lib/editor/create-editor.ts index 52e28dfc..c2922a07 100644 --- a/apps/editor/src/lib/editor/create-editor.ts +++ b/apps/editor/src/lib/editor/create-editor.ts @@ -49,6 +49,7 @@ import { registerCorePanels } from '$lib/project/register-core-panels'; import { followProjectCode } from '../code/follow-project-code'; import { followProjectPlugins, toDescriptor } from '../plugin/follow-project-plugins'; +import { followProjectCaches } from '../project/follow-project-caches'; import { followProjectTrust } from '../project/follow-project-trust'; import { followProjectRuntime } from '../runtime/follow-project-runtime'; import { WorkspaceActions, WorkspaceActionsToken } from '../workspace/workspace-actions'; @@ -136,6 +137,7 @@ export const createEditor = async ( ); const { settings } = settingsSetup; services.provide(AccountSyncToken, settingsSetup.account); + store.add(followProjectCaches({ services, projects, settings, session })); const history = store.add( new HistoryService({ diff --git a/apps/editor/src/lib/project/follow-project-caches.ts b/apps/editor/src/lib/project/follow-project-caches.ts new file mode 100644 index 00000000..686ba7ef --- /dev/null +++ b/apps/editor/src/lib/project/follow-project-caches.ts @@ -0,0 +1,60 @@ +import type { Container, Disposable } from '@nanoforge-dev/editor-kernel'; +import { + IndexedDbTreeCache, + OpfsContentCache, + ProjectCaches, + ProjectCachesToken, + type ProjectService, + removeOtherUsers, + removeStaleOpfs, +} from '@nanoforge-dev/editor-project'; +import type { SessionInfo } from '@nanoforge-dev/editor-protocol'; +import { CoreSettings, type SettingsService } from '@nanoforge-dev/editor-settings'; + +/** Upkeep runs this long after a project opens, when the start is over. */ +const SWEEP_DELAY_MS = 15_000; + +export interface FollowProjectCachesOptions { + readonly services: Container; + readonly projects: ProjectService; + readonly settings: SettingsService; + readonly session: SessionInfo; +} + +/** + * Keeps the files of opened projects in the browser (OPFS and IndexedDB), per user, within the + * Storage settings. A hosted editor removes what other users left on this browser (a shared + * computer). Without the storage APIs, files are only kept while the page lives. + */ +export const followProjectCaches = ({ + services, + projects, + settings, + session, +}: FollowProjectCachesOptions): Disposable => { + if (!OpfsContentCache.isSupported() || !IndexedDbTreeCache.isSupported()) + return { dispose: () => undefined }; + const user = session.user?.id ?? 'local'; + const online = session.mode === 'ONLINE'; + void removeStaleOpfs(online ? user : undefined); + if (online) void removeOtherUsers(user); + const caches = new ProjectCaches(user, new OpfsContentCache(user), { + budgetBytes: () => settings.get(CoreSettings.cacheBudget) * 1024 * 1024, + maxAgeDays: () => settings.get(CoreSettings.cacheMaxAge), + }); + projects.useCaches(caches); + services.provide(ProjectCachesToken, caches); + let timer: ReturnType | undefined; + const unsubscribe = projects.current.subscribe((project) => { + clearTimeout(timer); + if (project) + timer = setTimeout(() => void caches.sweep().catch(() => undefined), SWEEP_DELAY_MS); + }); + return { + dispose: () => { + clearTimeout(timer); + unsubscribe(); + caches.dispose(); + }, + }; +}; diff --git a/apps/editor/src/routes/(editor)/+page.svelte b/apps/editor/src/routes/(editor)/+page.svelte index df75e68f..43ef1099 100644 --- a/apps/editor/src/routes/(editor)/+page.svelte +++ b/apps/editor/src/routes/(editor)/+page.svelte @@ -12,6 +12,8 @@ Switch, } from '@nanoforge-dev/editor-ui'; + import { ProjectCachesToken } from '@nanoforge-dev/editor-project'; + import { getEditor } from '$lib/editor/editor-context'; const editor = getEditor(); @@ -81,6 +83,7 @@ try { await projectsApi.forget({ id: project.id }); recent = recent?.filter((candidate) => candidate.id !== project.id); + void editor.services.tryGet(ProjectCachesToken)?.clearProject(project.id); } catch (reason) { error = reason instanceof Error ? reason.message : String(reason); } diff --git a/docs/docs/editor/settings.mdx b/docs/docs/editor/settings.mdx index ee110c32..05571e78 100644 --- a/docs/docs/editor/settings.mdx +++ b/docs/docs/editor/settings.mdx @@ -25,6 +25,14 @@ When several scopes have a value, the more specific one wins: Project (only me), _Edit as JSON_ opens the project's settings file in the code editor, with completion. +## Files kept in the browser + +The editor keeps the files of the projects you open in your browser, so opening one again downloads only what changed. A file that several projects or branches have is kept once. + +_Editor › Storage_ shows the space each project takes, with **Clear** for one project and **Clear all**. **Cache size** (500 MB) and **Forget projects after** (30 days) are saved on this machine: the least recently used files go first, and the editor never takes more than a tenth of what the browser allows. Removing a project from _Recent_ also clears its files. Clearing only drops these copies: unsaved changes and undo history are kept apart and never cleared. Switch on **Keep these files when the browser runs short of space** so the browser does not remove them on its own. + +In the online editor, signing in as someone else on the same browser removes the previous user's files. + ## Import and export The _Import & export_ page saves a scope to a `.json` file, and loads one with a preview of what would change. diff --git a/packages/code/src/service/code-service.ts b/packages/code/src/service/code-service.ts index 6e78fa85..68461b9e 100644 --- a/packages/code/src/service/code-service.ts +++ b/packages/code/src/service/code-service.ts @@ -324,6 +324,14 @@ export class CodeService implements Disposable { const paths = this._options.fs.entries.filter( (entry) => entry.kind === 'file' && isCodeFile(entry.path), ); + /** Fills the cache in a few requests; the reads below then cost nothing. */ + await this._options.fs + .readMany( + paths + .filter((entry) => !this._options.documents.get(entry.path)) + .map((entry) => entry.path), + ) + .catch(() => undefined); for (let i = 0; i < paths.length; i += 32) { const batch = await Promise.all( paths.slice(i, i + 32).map(async (entry) => ({ diff --git a/packages/project/package.json b/packages/project/package.json index 9b1296c3..a95bdd2f 100644 --- a/packages/project/package.json +++ b/packages/project/package.json @@ -36,7 +36,7 @@ "lint": "prettier --check . && eslint --format=pretty .", "format": "prettier --write . && eslint --fix --format=pretty src", "typecheck": "tsc -p tsconfig.json", - "test:unit": "vitest run --config ../../vitest.config.ts" + "test:unit": "vitest run --config vitest.config.ts" }, "dependencies": { "@nanoforge-dev/editor-kernel": "workspace:*", @@ -46,6 +46,8 @@ }, "devDependencies": { "@nanoforge-dev/editor-server-core": "workspace:*", + "@vitest/browser-playwright": "catalog:test", + "playwright": "catalog:test", "typescript": "catalog:build", "vitest": "catalog:test" }, diff --git a/packages/project/src/cache/cache-db.ts b/packages/project/src/cache/cache-db.ts new file mode 100644 index 00000000..1a4d8c01 --- /dev/null +++ b/packages/project/src/cache/cache-db.ts @@ -0,0 +1,81 @@ +/** + * The editor's IndexedDB database of cached projects. Every store is created here, with one + * version: opening it elsewhere at another version would fail (`VersionError`). + */ +export const CACHE_DB = 'nanoforge-editor'; +const VERSION = 2; + +/** `user\0projectId` → the project's file tree. */ +export const TREES = 'trees'; +/** sha1 → {@link BlobRecord}: size and last use of each cached content. */ +export const BLOBS = 'blobs'; +/** `user\0projectId` → {@link ProjectRecord}. */ +export const PROJECTS = 'projects'; + +export interface BlobRecord { + readonly user: string; + readonly hash: string; + readonly size: number; + /** Last read or write, in ms (written at most once a minute). */ + readonly used: number; +} + +export interface ProjectRecord { + readonly user: string; + readonly projectId: string; + /** Last time the project was opened, in ms. */ + readonly opened: number; +} + +export const projectKey = (user: string, projectId: string) => `${user}\0${projectId}`; +export const blobKey = (user: string, hash: string) => `${user}\0${hash}`; + +export const isCacheDbSupported = (): boolean => typeof indexedDB !== 'undefined'; + +/** Opens the database; version 1 (trees by project id only) is dropped on upgrade. */ +export const openCacheDb = (): Promise => + new Promise((resolve, reject) => { + const request = indexedDB.open(CACHE_DB, VERSION); + request.onupgradeneeded = (event) => { + const db = request.result; + if (event.oldVersion < 2 && db.objectStoreNames.contains(TREES)) db.deleteObjectStore(TREES); + for (const store of [TREES, BLOBS, PROJECTS]) + if (!db.objectStoreNames.contains(store)) db.createObjectStore(store); + }; + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error ?? new Error('IndexedDB unavailable')); + request.onblocked = () => reject(new Error('IndexedDB upgrade blocked by another tab')); + }); + +/** Runs requests in one transaction; resolves with the last request's result. */ +export const transact = async ( + stores: string | string[], + mode: IDBTransactionMode, + run: (transaction: IDBTransaction) => IDBRequest | IDBRequest[] | undefined, +): Promise => { + const db = await openCacheDb(); + try { + return await new Promise((resolve, reject) => { + const transaction = db.transaction(stores, mode); + const requests = run(transaction); + const last = Array.isArray(requests) ? requests.at(-1) : requests; + transaction.oncomplete = () => resolve(last?.result as T); + transaction.onerror = () => reject(transaction.error ?? new Error('IndexedDB failed')); + transaction.onabort = () => reject(transaction.error ?? new Error('IndexedDB aborted')); + }); + } finally { + db.close(); + } +}; + +/** Every value of a store whose key starts with `prefix`. */ +export const valuesWithPrefix = (store: string, prefix: string): Promise => + transact(store, 'readonly', (transaction) => + transaction.objectStore(store).getAll(IDBKeyRange.bound(prefix, `${prefix}￿`)), + ); + +/** Every key of a store that starts with `prefix`. */ +export const keysWithPrefix = (store: string, prefix: string): Promise => + transact(store, 'readonly', (transaction) => + transaction.objectStore(store).getAllKeys(IDBKeyRange.bound(prefix, `${prefix}￿`)), + ).then((keys) => keys.map(String)); diff --git a/packages/project/src/cache/cache.type.ts b/packages/project/src/cache/cache.type.ts index 3de7b030..2fc6e777 100644 --- a/packages/project/src/cache/cache.type.ts +++ b/packages/project/src/cache/cache.type.ts @@ -1,20 +1,16 @@ import type { FileEntry } from '@nanoforge-dev/editor-protocol'; -export interface CachedContent { - readonly content: Uint8Array; - readonly mtime: number; - readonly hash: string; -} - -/** File contents cache (OPFS in browsers). Keys are project-relative paths. */ +/** + * File contents by the sha1 of their bytes (OPFS in browsers): each content is kept once, for + * every project and branch that has it; the tree (`FileEntry.hash`) says which one a path holds. + */ export interface ContentCache { - get(path: string): Promise; - set(path: string, value: CachedContent): Promise; - delete(path: string): Promise; - clear(): Promise; + /** The bytes of a content, or undefined when they are not kept (or do not match the hash). */ + get(hash: string): Promise; + set(hash: string, content: Uint8Array): Promise; } -/** Last known file tree (IndexedDB in browsers), for fast startup before revalidation. */ +/** Last known file tree of a project (IndexedDB in browsers), for a fast start and as the index. */ export interface TreeCache { load(): Promise; save(entries: readonly FileEntry[]): Promise; diff --git a/packages/project/src/cache/index.ts b/packages/project/src/cache/index.ts index e90bf1b2..ddfb4306 100644 --- a/packages/project/src/cache/index.ts +++ b/packages/project/src/cache/index.ts @@ -1,5 +1,6 @@ -export { OpfsContentCache } from './opfs.content-cache'; +export { OpfsContentCache, removeStaleOpfs } from './opfs.content-cache'; export { IndexedDbTreeCache } from './indexed-db.tree-cache'; export * from './cache.type'; export * from './memory.content-cache'; export * from './memory.tree-cache'; +export * from './project-caches'; diff --git a/packages/project/src/cache/indexed-db.tree-cache.ts b/packages/project/src/cache/indexed-db.tree-cache.ts index 590c68f5..e8c4f523 100644 --- a/packages/project/src/cache/indexed-db.tree-cache.ts +++ b/packages/project/src/cache/indexed-db.tree-cache.ts @@ -1,22 +1,24 @@ import type { FileEntry } from '@nanoforge-dev/editor-protocol'; +import { TREES, isCacheDbSupported, projectKey, transact } from './cache-db'; import type { TreeCache } from './cache.type'; -export const DB_NAME = 'nanoforge-editor'; -export const STORE = 'trees'; - -/** IndexedDB tree cache (one record per project). */ +/** IndexedDB tree cache (one record per user and project). Failures degrade to misses. */ export class IndexedDbTreeCache implements TreeCache { + private readonly _key: string; + static isSupported(): boolean { - return typeof indexedDB !== 'undefined'; + return isCacheDbSupported(); } - constructor(private readonly _projectId: string) {} + constructor(user: string, projectId: string) { + this._key = projectKey(user, projectId); + } async load(): Promise { try { - return await this._run('readonly', (store) => - store.get(this._projectId), + return await transact(TREES, 'readonly', (transaction) => + transaction.objectStore(TREES).get(this._key), ); } catch { return undefined; @@ -25,28 +27,9 @@ export class IndexedDbTreeCache implements TreeCache { async save(entries: readonly FileEntry[]): Promise { try { - await this._run('readwrite', (store) => store.put(entries, this._projectId)); + await transact(TREES, 'readwrite', (transaction) => + transaction.objectStore(TREES).put(entries, this._key), + ); } catch {} } - - private async _run( - mode: IDBTransactionMode, - action: (store: IDBObjectStore) => IDBRequest, - ): Promise { - const db = await new Promise((resolve, reject) => { - const request = indexedDB.open(DB_NAME, 1); - request.onupgradeneeded = () => request.result.createObjectStore(STORE); - request.onsuccess = () => resolve(request.result); - request.onerror = () => reject(request.error ?? new Error('IndexedDB unavailable')); - }); - try { - return await new Promise((resolve, reject) => { - const request = action(db.transaction(STORE, mode).objectStore(STORE)); - request.onsuccess = () => resolve(request.result as T); - request.onerror = () => reject(request.error ?? new Error('IndexedDB request failed')); - }); - } finally { - db.close(); - } - } } diff --git a/packages/project/src/cache/memory.content-cache.ts b/packages/project/src/cache/memory.content-cache.ts index 94d42f3e..69e3239a 100644 --- a/packages/project/src/cache/memory.content-cache.ts +++ b/packages/project/src/cache/memory.content-cache.ts @@ -1,26 +1,14 @@ -import type { CachedContent, ContentCache } from './cache.type'; +import type { ContentCache } from './cache.type'; export class MemoryContentCache implements ContentCache { - private readonly _entries = new Map(); + private readonly _contents = new Map(); - get(path: string) { - return Promise.resolve(this._entries.get(path)); + get(hash: string) { + return Promise.resolve(this._contents.get(hash)); } - set(path: string, value: CachedContent) { - this._entries.set(path, value); - return Promise.resolve(); - } - - delete(path: string) { - for (const key of [...this._entries.keys()]) { - if (key === path || key.startsWith(`${path}/`)) this._entries.delete(key); - } - return Promise.resolve(); - } - - clear() { - this._entries.clear(); + set(hash: string, content: Uint8Array) { + this._contents.set(hash, content); return Promise.resolve(); } } diff --git a/packages/project/src/cache/opfs-write.ts b/packages/project/src/cache/opfs-write.ts new file mode 100644 index 00000000..48aac957 --- /dev/null +++ b/packages/project/src/cache/opfs-write.ts @@ -0,0 +1,87 @@ +/** + * Writes a file of the origin private file system. `createWritable` is missing in some browsers + * (Safari); there, a worker writes with `createSyncAccessHandle`, which only workers have. + */ +export const writeOpfsFile = async ( + path: readonly string[], + bytes: Uint8Array, + /** Tests: writes through the worker even where `createWritable` exists. */ + options: { readonly viaWorker?: boolean } = {}, +): Promise => { + const handle = await fileHandle(path); + if ( + !options.viaWorker && + 'createWritable' in handle && + typeof handle.createWritable === 'function' + ) { + const writable = await handle.createWritable(); + try { + await writable.write(bytes as Uint8Array); + } finally { + await writable.close(); + } + return; + } + await writeInWorker(path, bytes); +}; + +/** Whether files can be written here: `createWritable`, or workers for the fallback. */ +export const canWriteOpfs = (): boolean => + typeof FileSystemFileHandle !== 'undefined' && + ('createWritable' in FileSystemFileHandle.prototype || typeof Worker !== 'undefined'); + +const fileHandle = async (path: readonly string[]): Promise => { + let dir = await navigator.storage.getDirectory(); + for (const part of path.slice(0, -1)) dir = await dir.getDirectoryHandle(part, { create: true }); + return dir.getFileHandle(path.at(-1)!, { create: true }); +}; + +const WORKER_SOURCE = ` +onmessage = async ({ data: { id, path, bytes } }) => { + try { + let dir = await navigator.storage.getDirectory(); + for (const part of path.slice(0, -1)) dir = await dir.getDirectoryHandle(part, { create: true }); + const handle = await dir.getFileHandle(path[path.length - 1], { create: true }); + const access = await handle.createSyncAccessHandle(); + try { + access.truncate(0); + access.write(bytes, { at: 0 }); + access.flush(); + } finally { + access.close(); + } + postMessage({ id, ok: true }); + } catch (error) { + postMessage({ id, ok: false, name: error && error.name, message: String(error) }); + } +}; +`; + +let worker: Worker | undefined; +let nextId = 0; +const waiting = new Map void; reject: (error: Error) => void }>(); + +const writeInWorker = (path: readonly string[], bytes: Uint8Array): Promise => { + if (!worker) { + const url = URL.createObjectURL(new Blob([WORKER_SOURCE], { type: 'text/javascript' })); + worker = new Worker(url); + worker.onmessage = ({ + data, + }: MessageEvent<{ id: number; ok: boolean; name?: string; message?: string }>) => { + const pending = waiting.get(data.id); + waiting.delete(data.id); + if (data.ok) pending?.resolve(); + else + pending?.reject( + Object.assign(new Error(data.message ?? 'OPFS write failed'), { + name: data.name ?? 'Error', + }), + ); + }; + } + const id = nextId++; + return new Promise((resolve, reject) => { + waiting.set(id, { resolve, reject }); + worker!.postMessage({ id, path: [...path], bytes }); + }); +}; diff --git a/packages/project/src/cache/opfs.content-cache.ts b/packages/project/src/cache/opfs.content-cache.ts index 8971fe5d..fa706804 100644 --- a/packages/project/src/cache/opfs.content-cache.ts +++ b/packages/project/src/cache/opfs.content-cache.ts @@ -1,83 +1,98 @@ -import type { CachedContent, ContentCache } from './cache.type'; +import type { ContentCache } from './cache.type'; +import { canWriteOpfs, writeOpfsFile } from './opfs-write'; +import { sha1Hex } from './sha1'; + +/** The folder of the editor in the origin private file system. */ +export const OPFS_ROOT = 'nanoforge-editor'; +/** The layout of this version: `v2//blobs/`. Anything else in the root is older. */ +export const OPFS_LAYOUT = 'v2'; /** - * OPFS content cache: `nanoforge-editor//` holds the bytes and a - * small sidecar JSON holds mtime and hash. Failures degrade to cache misses. + * OPFS content cache of one user: `nanoforge-editor/v2//blobs/`. Every read checks + * the bytes against their hash, so a write cut short (or any mistake) is a miss, never the wrong + * content. Failures degrade to misses; `set` throws (`QuotaExceededError`) for the caller to + * make room. */ export class OpfsContentCache implements ContentCache { private _dir: Promise | undefined; static isSupported(): boolean { return ( - typeof navigator !== 'undefined' && typeof navigator.storage?.getDirectory === 'function' + typeof navigator !== 'undefined' && + typeof navigator.storage?.getDirectory === 'function' && + typeof crypto !== 'undefined' && + typeof crypto.subtle?.digest === 'function' && + canWriteOpfs() ); } - constructor(private readonly _projectId: string) {} + constructor(private readonly _user: string) {} - async get(path: string): Promise { + async get(hash: string): Promise { try { - const dir = await this._directory(); - const meta = await (await (await dir.getFileHandle(`${key(path)}.meta`)).getFile()).text(); - const bytes = await (await (await dir.getFileHandle(key(path))).getFile()).arrayBuffer(); - const { mtime, hash } = JSON.parse(meta) as { mtime: number; hash: string }; - return { content: new Uint8Array(bytes), mtime, hash }; - } catch { - return undefined; - } + const file = await (await (await this._directory()).getFileHandle(hash)).getFile(); + const bytes = new Uint8Array(await file.arrayBuffer()); + if ((await sha1Hex(bytes)) === hash) return bytes; + await this.delete(hash); + } catch {} + return undefined; } - async set(path: string, value: CachedContent): Promise { - try { - const dir = await this._directory(); - await write(dir, key(path), value.content); - await write( - dir, - `${key(path)}.meta`, - JSON.stringify({ mtime: value.mtime, hash: value.hash }), - ); - } catch {} + async set(hash: string, content: Uint8Array): Promise { + await writeOpfsFile([OPFS_ROOT, OPFS_LAYOUT, this._folder(), 'blobs', hash], content); } - async delete(path: string): Promise { + async delete(hash: string): Promise { try { - const dir = await this._directory(); - const prefix = key(path); - for await (const name of (dir as unknown as { keys(): AsyncIterable }).keys()) { - if (name === prefix || name === `${prefix}.meta` || name.startsWith(`${prefix}%2F`)) { - await dir.removeEntry(name); - } - } + await (await this._directory()).removeEntry(hash); } catch {} } + /** Removes every content of this user. */ async clear(): Promise { try { - const root = await ( + const layout = await ( await navigator.storage.getDirectory() - ).getDirectoryHandle('nanoforge-editor', { create: true }); - await root.removeEntry(this._projectId, { recursive: true }); - this._dir = undefined; + ) + .getDirectoryHandle(OPFS_ROOT) + .then((root) => root.getDirectoryHandle(OPFS_LAYOUT)); + await layout.removeEntry(this._folder(), { recursive: true }); } catch {} + this._dir = undefined; + } + + private _folder(): string { + return encodeURIComponent(this._user); } private _directory(): Promise { this._dir ??= navigator.storage .getDirectory() - .then((root) => root.getDirectoryHandle('nanoforge-editor', { create: true })) - .then((root) => root.getDirectoryHandle(this._projectId, { create: true })); + .then((root) => root.getDirectoryHandle(OPFS_ROOT, { create: true })) + .then((root) => root.getDirectoryHandle(OPFS_LAYOUT, { create: true })) + .then((root) => root.getDirectoryHandle(this._folder(), { create: true })) + .then((root) => root.getDirectoryHandle('blobs', { create: true })); return this._dir; } } -export const key = (path: string) => encodeURIComponent(path); - -export const write = async ( - dir: FileSystemDirectoryHandle, - name: string, - data: Uint8Array | string, -) => { - const writable = await (await dir.getFileHandle(name, { create: true })).createWritable(); - await writable.write(typeof data === 'string' ? data : new Blob([data as BlobPart])); - await writable.close(); +/** + * Removes what older versions left in the editor's OPFS folder (`/` folders of + * per-path copies), and the folders of other users when `keepUser` is given. + */ +export const removeStaleOpfs = async (keepUser?: string): Promise => { + try { + const root = await ( + await navigator.storage.getDirectory() + ).getDirectoryHandle(OPFS_ROOT, { + create: true, + }); + for await (const name of (root as unknown as { keys(): AsyncIterable }).keys()) + if (name !== OPFS_LAYOUT) await root.removeEntry(name, { recursive: true }).catch(() => {}); + if (keepUser === undefined) return; + const layout = await root.getDirectoryHandle(OPFS_LAYOUT, { create: true }); + for await (const name of (layout as unknown as { keys(): AsyncIterable }).keys()) + if (name !== encodeURIComponent(keepUser)) + await layout.removeEntry(name, { recursive: true }).catch(() => {}); + } catch {} }; diff --git a/packages/project/src/cache/project-caches.ts b/packages/project/src/cache/project-caches.ts new file mode 100644 index 00000000..5a870ab5 --- /dev/null +++ b/packages/project/src/cache/project-caches.ts @@ -0,0 +1,369 @@ +import { type Disposable, createToken } from '@nanoforge-dev/editor-kernel'; +import type { FileEntry } from '@nanoforge-dev/editor-protocol'; + +import type { CacheFactory } from '../service/client-project'; +import { + BLOBS, + type BlobRecord, + PROJECTS, + type ProjectRecord, + TREES, + blobKey, + keysWithPrefix, + projectKey, + transact, + valuesWithPrefix, +} from './cache-db'; +import type { ContentCache, TreeCache } from './cache.type'; +import { IndexedDbTreeCache } from './indexed-db.tree-cache'; +import { type OpfsContentCache } from './opfs.content-cache'; + +const DAY_MS = 24 * 60 * 60 * 1000; +/** A content written or read this recently is never swept: another tab may be about to use it. */ +const RECENT_MS = 10 * 60 * 1000; +/** Last-use times are written at most this often. */ +const TOUCH_DELAY_MS = 60 * 1000; +/** Contents read or written lately, kept in memory too: a project opening reads them twice. */ +const MEMORY_BYTES = 64 * 1024 * 1024; + +export interface ProjectCachesOptions { + /** Budget in bytes; the cache never takes more than a tenth of the browser's quota either. */ + readonly budgetBytes: () => number; + /** Projects not opened for this many days are dropped (0 keeps them). */ + readonly maxAgeDays: () => number; +} + +/** What the cache holds for a project. */ +export interface ProjectUsage { + readonly projectId: string; + readonly bytes: number; + readonly files: number; + readonly opened: number; +} + +export interface CacheUsage { + readonly projects: readonly ProjectUsage[]; + /** Every content kept, counted once even when several projects have it. */ + readonly bytes: number; + readonly budget: number; + /** Whether the browser keeps this storage when space runs short (`navigator.storage.persist`). */ + readonly persisted: boolean; +} + +/** + * The browser caches of one user's projects: a content store shared by every project (by + * sha1), each project's tree, and their upkeep. Contents beyond the budget go least recently + * used first, projects not opened for a while are dropped, and contents no tree uses are swept. + * Projects open in a tab hold a lock that keeps their contents. + */ +export class ProjectCaches implements CacheFactory { + private readonly _touched = new Map(); + private readonly _memory = new Map(); + private _memoryBytes = 0; + /** Bytes written since the last upkeep: past a tenth of the budget, upkeep runs again. */ + private _writtenSinceSweep = 0; + private _sweepTimer: ReturnType | undefined; + private _touchTimer: ReturnType | undefined; + private readonly _contents: ContentCache; + + constructor( + readonly user: string, + private readonly _store: OpfsContentCache, + private readonly _options: ProjectCachesOptions, + ) { + this._contents = { + get: async (hash) => { + const bytes = this._memory.get(hash) ?? (await this._store.get(hash)); + if (bytes) { + this._touch(hash, bytes.length); + this._remember(hash, bytes); + } + return bytes; + }, + set: async (hash, content) => { + this._remember(hash, content); + await this._set(hash, content); + }, + }; + } + + contents(): ContentCache { + return this._contents; + } + + tree(projectId: string): TreeCache { + return new IndexedDbTreeCache(this.user, projectId); + } + + /** Records that a project opened, and keeps its contents while the returned lease lives. */ + opened(projectId: string): Disposable { + void transact(PROJECTS, 'readwrite', (transaction) => + transaction + .objectStore(PROJECTS) + .put( + { user: this.user, projectId, opened: Date.now() } satisfies ProjectRecord, + projectKey(this.user, projectId), + ), + ).catch(() => undefined); + let release: () => void = () => undefined; + const locks = typeof navigator !== 'undefined' ? navigator.locks : undefined; + if (locks) { + const held = new Promise((resolve) => (release = resolve)); + void locks + .request(this._projectLock(projectId), { mode: 'shared' }, () => held) + .catch(() => undefined); + } + return { dispose: () => release() }; + } + + async usage(): Promise { + await this._flushTouches(); + const [projects, trees, blobs, budget, persisted] = await Promise.all([ + valuesWithPrefix(PROJECTS, `${this.user}\0`), + this._trees(), + valuesWithPrefix(BLOBS, `${this.user}\0`), + this._budget(), + navigator.storage?.persisted?.().catch(() => false) ?? Promise.resolve(false), + ]); + const sizes = new Map(blobs.map((blob) => [blob.hash, blob.size])); + return { + projects: projects.map((project) => { + const hashes = new Set( + (trees.get(project.projectId) ?? []).flatMap((entry) => entry.hash ?? []), + ); + let bytes = 0; + let files = 0; + for (const hash of hashes) { + const size = sizes.get(hash); + if (size === undefined) continue; + bytes += size; + files++; + } + return { projectId: project.projectId, bytes, files, opened: project.opened }; + }), + bytes: blobs.reduce((sum, blob) => sum + blob.size, 0), + budget, + persisted, + }; + } + + /** Drops a project's tree, and the contents no other project uses. */ + async clearProject(projectId: string): Promise { + await transact([TREES, PROJECTS], 'readwrite', (transaction) => [ + transaction.objectStore(TREES).delete(projectKey(this.user, projectId)), + transaction.objectStore(PROJECTS).delete(projectKey(this.user, projectId)), + ]).catch(() => undefined); + await this.sweep({ recentToo: true }); + } + + /** Drops everything this user has cached. */ + async clearAll(): Promise { + await this._exclusive(async () => { + this._touched.clear(); + this._memory.clear(); + this._memoryBytes = 0; + await this._store.clear(); + const prefix = `${this.user}\0`; + for (const store of [TREES, BLOBS, PROJECTS]) { + const keys = await keysWithPrefix(store, prefix).catch(() => []); + await transact(store, 'readwrite', (transaction) => + keys.map((key) => transaction.objectStore(store).delete(key)), + ).catch(() => undefined); + } + }); + } + + /** + * Upkeep: drops projects not opened for `maxAgeDays`, the contents no tree uses, then the least + * recently used contents of closed projects until `room` more bytes fit in the budget. + */ + async sweep(options: { room?: number; recentToo?: boolean } = {}): Promise { + this._writtenSinceSweep = 0; + clearTimeout(this._sweepTimer); + this._sweepTimer = undefined; + await this._exclusive(async () => { + await this._flushTouches(); + const now = Date.now(); + const open = await this._openProjects(); + const maxAge = this._options.maxAgeDays(); + if (maxAge > 0) { + const projects = await valuesWithPrefix(PROJECTS, `${this.user}\0`); + const expired = projects.filter( + (project) => !open.has(project.projectId) && project.opened < now - maxAge * DAY_MS, + ); + if (expired.length) + await transact([TREES, PROJECTS], 'readwrite', (transaction) => + expired.flatMap((project) => [ + transaction.objectStore(TREES).delete(projectKey(this.user, project.projectId)), + transaction.objectStore(PROJECTS).delete(projectKey(this.user, project.projectId)), + ]), + ); + } + const trees = await this._trees(); + const used = new Set(); + const kept = new Set(); + for (const [projectId, entries] of trees) + for (const entry of entries) + if (entry.hash) { + used.add(entry.hash); + if (open.has(projectId)) kept.add(entry.hash); + } + const blobs = await valuesWithPrefix(BLOBS, `${this.user}\0`); + const recent = (blob: BlobRecord) => !options.recentToo && blob.used > now - RECENT_MS; + const gone = blobs.filter((blob) => !used.has(blob.hash) && !recent(blob)); + let total = blobs.reduce((sum, blob) => sum + blob.size, 0); + for (const blob of gone) total -= blob.size; + const budget = await this._budget(); + const target = budget - (options.room ?? 0); + if (total > target) { + const candidates = blobs + .filter((blob) => used.has(blob.hash) && !kept.has(blob.hash) && !recent(blob)) + .sort((a, b) => a.used - b.used); + for (const blob of candidates) { + if (total <= target) break; + gone.push(blob); + total -= blob.size; + } + } + for (const blob of gone) { + await this._store.delete(blob.hash); + const kept = this._memory.get(blob.hash); + if (kept) { + this._memory.delete(blob.hash); + this._memoryBytes -= kept.length; + } + } + if (gone.length) + await transact(BLOBS, 'readwrite', (transaction) => + gone.map((blob) => transaction.objectStore(BLOBS).delete(blobKey(this.user, blob.hash))), + ); + }); + } + + dispose(): void { + clearTimeout(this._sweepTimer); + void this._flushTouches(); + } + + private async _set(hash: string, content: Uint8Array): Promise { + try { + await this._store.set(hash, content); + } catch (error) { + if ((error as { name?: string })?.name !== 'QuotaExceededError') return; + await this.sweep({ room: content.length }).catch(() => undefined); + try { + await this._store.set(hash, content); + } catch { + return; + } + } + await transact(BLOBS, 'readwrite', (transaction) => + transaction + .objectStore(BLOBS) + .put( + { user: this.user, hash, size: content.length, used: Date.now() } satisfies BlobRecord, + blobKey(this.user, hash), + ), + ).catch(() => undefined); + this._writtenSinceSweep += content.length; + if (!this._sweepTimer && this._writtenSinceSweep > this._options.budgetBytes() / 10) + this._sweepTimer = setTimeout(() => void this.sweep().catch(() => undefined), 30_000); + } + + private _remember(hash: string, bytes: Uint8Array): void { + if (bytes.length > MEMORY_BYTES / 4) return; + const known = this._memory.get(hash); + if (known) { + this._memory.delete(hash); + this._memory.set(hash, known); + return; + } + this._memory.set(hash, bytes); + this._memoryBytes += bytes.length; + for (const [oldest, old] of this._memory) { + if (this._memoryBytes <= MEMORY_BYTES) break; + this._memory.delete(oldest); + this._memoryBytes -= old.length; + } + } + + private _touch(hash: string, size: number): void { + this._touched.set(hash, size); + this._touchTimer ??= setTimeout(() => void this._flushTouches(), TOUCH_DELAY_MS); + } + + private async _flushTouches(): Promise { + clearTimeout(this._touchTimer); + this._touchTimer = undefined; + if (!this._touched.size) return; + const touched = [...this._touched]; + this._touched.clear(); + const now = Date.now(); + await transact(BLOBS, 'readwrite', (transaction) => + touched.map(([hash, size]) => + transaction + .objectStore(BLOBS) + .put( + { user: this.user, hash, size, used: now } satisfies BlobRecord, + blobKey(this.user, hash), + ), + ), + ).catch(() => undefined); + } + + private async _trees(): Promise> { + const prefix = `${this.user}\0`; + const keys = await keysWithPrefix(TREES, prefix); + const values = await valuesWithPrefix(TREES, prefix); + return new Map(keys.map((key, index) => [key.slice(prefix.length), values[index] ?? []])); + } + + private async _budget(): Promise { + const quota = (await navigator.storage?.estimate?.().catch(() => undefined))?.quota; + const budget = this._options.budgetBytes(); + return quota ? Math.min(budget, quota * 0.1) : budget; + } + + private _projectLock(projectId: string): string { + return `nanoforge-project:${this.user}:${projectId}`; + } + + /** Projects open in a tab of this browser (they hold their lock). */ + private async _openProjects(): Promise> { + const locks = typeof navigator !== 'undefined' ? navigator.locks : undefined; + const prefix = `nanoforge-project:${this.user}:`; + const held = (await locks?.query().catch(() => undefined))?.held ?? []; + return new Set( + held.flatMap((lock) => + lock.name?.startsWith(prefix) ? [lock.name.slice(prefix.length)] : [], + ), + ); + } + + private async _exclusive(run: () => Promise): Promise { + const locks = typeof navigator !== 'undefined' ? navigator.locks : undefined; + if (!locks) return run(); + await locks.request(`nanoforge-cache-sweep:${this.user}`, run); + } +} + +/** + * Removes what other users cached in this browser (a hosted editor on a shared computer): their + * trees, records and contents. + */ +export const removeOtherUsers = async (user: string): Promise => { + const own = `${user}\0`; + for (const store of [TREES, BLOBS, PROJECTS]) { + const keys = await transact(store, 'readonly', (transaction) => + transaction.objectStore(store).getAllKeys(), + ).catch(() => [] as IDBValidKey[]); + const others = keys.map(String).filter((key) => !key.startsWith(own)); + if (others.length) + await transact(store, 'readwrite', (transaction) => + others.map((key) => transaction.objectStore(store).delete(key)), + ).catch(() => undefined); + } +}; + +/** The browser caches of the signed-in user, when this browser can keep them. */ +export const ProjectCachesToken = createToken('project.caches'); diff --git a/packages/project/src/cache/sha1.ts b/packages/project/src/cache/sha1.ts new file mode 100644 index 00000000..2062fdbf --- /dev/null +++ b/packages/project/src/cache/sha1.ts @@ -0,0 +1,5 @@ +/** sha1 of bytes as hex, like the server's (`crypto.subtle`, native in browsers and Node). */ +export const sha1Hex = async (bytes: Uint8Array): Promise => { + const digest = await crypto.subtle.digest('SHA-1', bytes as Uint8Array); + return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, '0')).join(''); +}; diff --git a/packages/project/src/fs/project-fs.ts b/packages/project/src/fs/project-fs.ts index 21f6e9a9..75f5a9b4 100644 --- a/packages/project/src/fs/project-fs.ts +++ b/packages/project/src/fs/project-fs.ts @@ -12,9 +12,10 @@ import { type FileEntry, FsContract, } from '@nanoforge-dev/editor-protocol'; -import type { RpcApi, RpcClient } from '@nanoforge-dev/editor-rpc'; +import { type RpcApi, type RpcClient, RpcError } from '@nanoforge-dev/editor-rpc'; import type { ContentCache, TreeCache } from '../cache/cache.type'; +import { sha1Hex } from '../cache/sha1'; const decoder = new TextDecoder(); @@ -26,10 +27,15 @@ export interface WriteOptions { expectedHash?: string | null; } +/** Whether two entries hold the same content (by hash when both have one). */ +const sameContent = (a: FileEntry, b: FileEntry): boolean => + a.hash && b.hash ? a.hash === b.hash : a.mtime === b.mtime && a.size === b.size; + /** * Files of the open project. The tree is loaded once (cached for fast startup), then kept in - * sync by the server change stream, so contents can be served from the local cache whenever - * the cached copy is as recent as the tree says. + * sync by the server change stream. Contents come from the local cache, keyed by the hash the + * tree gives each file: a copy is served when its hash is the tree's, whatever the dates say, + * and a renamed or copied file is already there. */ export class ProjectFs implements Disposable { private readonly _fs: RpcApi; @@ -37,6 +43,7 @@ export class ProjectFs implements Disposable { private readonly _ready = new ObservableValue(false); private readonly _onDidChange = new Emitter(); private readonly _store = new DisposableStore(); + private _treeSave: ReturnType | undefined; /** Batched changes (from the server watcher and this client's own writes). */ readonly onDidChange: Event = this._onDidChange.event; @@ -79,7 +86,7 @@ export class ProjectFs implements Disposable { for (const [path, entry] of next) { const previous = this._tree.get(path); if (!previous) changes.push({ type: 'created', path, kind: entry.kind }); - else if (previous.mtime !== entry.mtime || previous.size !== entry.size) { + else if (entry.kind === 'file' && !sameContent(previous, entry)) { changes.push({ type: 'changed', path, kind: entry.kind }); } } @@ -90,8 +97,6 @@ export class ProjectFs implements Disposable { for (const [path, entry] of next) this._tree.set(path, entry); this._ready.set(true); await this._treeCache.save(entries); - for (const change of changes) - if (change.type === 'deleted') await this._contents.delete(change.path); if (changes.length) this._onDidChange.fire(changes); } @@ -121,18 +126,29 @@ export class ProjectFs implements Disposable { /** * `fresh` asks the server even when the cached content looks current: the tree only learns - * of a change on disk when the watcher reports it. + * of a change on disk when the watcher reports it. The server then sends the bytes only if the + * cached copy is out of date. */ async read(path: string, options: { fresh?: boolean } = {}): Promise { - const known = this._tree.get(path); - const cached = !options.fresh && known && (await this._contents.get(path)); - if (known && cached && cached.mtime === known.mtime && cached.content.length === known.size) { - return { ...known, content: cached.content, hash: cached.hash }; - } - const file = await this._fs.read({ project: this.projectId, path }); - this._tree.set(path, { path, kind: 'file', size: file.size, mtime: file.mtime }); - await this._contents.set(path, { content: file.content, mtime: file.mtime, hash: file.hash }); - return file; + const [result] = await this._fetch([path], options.fresh ?? false); + if (!result || 'error' in result) + throw new RpcError( + (result?.error.code ?? 'INTERNAL') as RpcError['code'], + result?.error.message ?? `Could not read ${path}`, + ); + return result; + } + + /** + * Reads several files at once (a project opening, open tabs, type information): the ones the + * cache has cost nothing, the others come in a few batched requests. Unreadable files are left + * out. + */ + async readMany(paths: readonly string[]): Promise> { + const results = await this._fetch(paths, false); + return new Map( + results.flatMap((result) => ('error' in result ? [] : [[result.path, result] as const])), + ); } async readText( @@ -156,8 +172,17 @@ export class ProjectFs implements Disposable { }); const bytes = typeof content === 'string' ? new TextEncoder().encode(content) : content; const existed = this._tree.has(path); - this._tree.set(path, { path, kind: 'file', size: result.size, mtime: result.mtime }); - await this._contents.set(path, { content: bytes, mtime: result.mtime, hash: result.hash }); + this._tree.set(path, { + path, + kind: 'file', + size: result.size, + mtime: result.mtime, + hash: result.hash, + }); + /** Kept only if the server wrote exactly these bytes: the cache must never lie. */ + if ((await sha1Hex(bytes).catch(() => undefined)) === result.hash) + await this._contents.set(result.hash, bytes).catch(() => undefined); + this._scheduleTreeSave(); this._onDidChange.fire([{ type: existed ? 'changed' : 'created', path, kind: 'file' }]); return result; } @@ -182,7 +207,7 @@ export class ProjectFs implements Disposable { const kind = this.entry(path)?.kind ?? 'file'; const result = await this._fs.delete({ project: this.projectId, path }); this._removeFromTree(path); - await this._contents.delete(path); + this._scheduleTreeSave(); this._onDidChange.fire([{ type: 'deleted', path, kind }]); return result; } @@ -199,6 +224,10 @@ export class ProjectFs implements Disposable { } dispose(): void { + if (this._treeSave) { + clearTimeout(this._treeSave); + void this._treeCache.save([...this._tree.values()]); + } this._store.dispose(); this._onDidChange.dispose(); } @@ -208,7 +237,6 @@ export class ProjectFs implements Disposable { for (const change of changes) { if (change.type === 'deleted') { this._removeFromTree(change.path); - await this._contents.delete(change.path); relevant.push(change); continue; } @@ -218,13 +246,104 @@ export class ProjectFs implements Disposable { if (!entry) continue; const previous = this._tree.get(change.path); this._tree.set(change.path, entry); - if (previous && previous.mtime === entry.mtime && previous.size === entry.size) continue; + if (previous && entry.kind === 'file' && sameContent(previous, entry)) continue; relevant.push(change); } + this._scheduleTreeSave(); if (relevant.length) this._onDidChange.fire(relevant); if (changes.some((change) => change.path === NFIGNORE_FILE)) await this.refresh(); } + /** + * Contents of files: from the cache when the tree's hash is there (unless `fresh`), else from + * the server, which answers `notModified` for a cached copy still current. + */ + private async _fetch( + paths: readonly string[], + fresh: boolean, + ): Promise<(FileContent | { path: string; error: { code: string; message: string } })[]> { + const found = new Map(); + const cached = new Map(); + const ask: { path: string; knownHash?: string }[] = []; + /** Cache lookups run together: each one is a read of the browser's storage. */ + const lookups = await Promise.all( + paths.map(async (path) => { + const known = this._tree.get(path); + const bytes = known?.hash ? await this._contents.get(known.hash) : undefined; + return { path, known, bytes }; + }), + ); + for (const { path, known, bytes } of lookups) { + if (known?.hash && bytes) { + if (!fresh) { + found.set(path, { ...known, content: bytes, hash: known.hash }); + continue; + } + cached.set(path, bytes); + ask.push({ path, knownHash: known.hash }); + } else ask.push({ path }); + } + const errors = new Map(); + /** Chunks of 64 files, 4 requests at a time, like reads one by one would overlap. */ + const chunks: (typeof ask)[] = []; + for (let start = 0; start < ask.length; start += 64) chunks.push(ask.slice(start, start + 64)); + const readChunk = async (chunk: typeof ask) => { + let pending = chunk; + while (pending.length) { + const answer = await this._fs.readMany({ project: this.projectId, files: pending }); + for (const result of answer.files) { + if ('error' in result) { + errors.set(result.path, result.error); + continue; + } + const entry: FileEntry = { + path: result.path, + kind: 'file', + size: result.size, + mtime: result.mtime, + hash: result.hash, + }; + this._tree.set(result.path, entry); + if ('notModified' in result) { + const bytes = cached.get(result.path); + if (bytes) found.set(result.path, { ...entry, content: bytes, hash: result.hash }); + continue; + } + found.set(result.path, result); + /** Stored in the background: the bytes are already here, and the disk is slow. */ + void this._contents.set(result.hash, result.content).catch(() => undefined); + } + const rest = new Set(answer.rest); + pending = pending.filter((file) => rest.has(file.path)); + } + }; + const queue = [...chunks]; + const worker = async () => { + for (let chunk = queue.shift(); chunk; chunk = queue.shift()) await readChunk(chunk); + }; + await Promise.all(Array.from({ length: Math.min(4, queue.length) }, worker)); + if (ask.length) this._scheduleTreeSave(); + return paths.map( + (path) => + found.get(path) ?? { + path, + error: errors.get(path) ?? { code: 'NOT_FOUND', message: `Could not read ${path}` }, + }, + ); + } + + /** + * Saves the tree a moment after it changed in this session (writes, remote changes, reads): + * the cache's upkeep reads the saved trees to know which contents projects still use. + */ + private _scheduleTreeSave(): void { + clearTimeout(this._treeSave); + this._treeSave = setTimeout(() => { + this._treeSave = undefined; + void this._treeCache.save([...this._tree.values()]); + }, 1000); + } + private _removeFromTree(path: string): void { for (const key of [...this._tree.keys()]) { if (key === path || key.startsWith(`${path}/`)) this._tree.delete(key); diff --git a/packages/project/src/service/client-project.ts b/packages/project/src/service/client-project.ts index ada45ff3..00f9a34a 100644 --- a/packages/project/src/service/client-project.ts +++ b/packages/project/src/service/client-project.ts @@ -11,23 +11,22 @@ import { type ProjectModel, ProjectsContract } from '@nanoforge-dev/editor-proto import type { RpcClient } from '@nanoforge-dev/editor-rpc'; import type { ContentCache, TreeCache } from '../cache/cache.type'; -import { IndexedDbTreeCache } from '../cache/indexed-db.tree-cache'; import { MemoryContentCache } from '../cache/memory.content-cache'; import { MemoryTreeCache } from '../cache/memory.tree-cache'; -import { OpfsContentCache } from '../cache/opfs.content-cache'; import { ProjectFs } from '../fs/project-fs'; +/** Where a project's files are cached (`ProjectCaches` in browsers that can keep them). */ export interface CacheFactory { contents(projectId: string): ContentCache; tree(projectId: string): TreeCache; + /** Called when a project opens; the lease is disposed when it closes. */ + opened?(projectId: string): Disposable; } -/** Browser caches when available, memory otherwise. */ -export const defaultCacheFactory: CacheFactory = { - contents: (id) => - OpfsContentCache.isSupported() ? new OpfsContentCache(id) : new MemoryContentCache(), - tree: (id) => - IndexedDbTreeCache.isSupported() ? new IndexedDbTreeCache(id) : new MemoryTreeCache(), +/** Caches that live as long as the page (tests, browsers without storage). */ +export const memoryCacheFactory = (): CacheFactory => { + const contents = new MemoryContentCache(); + return { contents: () => contents, tree: () => new MemoryTreeCache() }; }; /** A project opened in this editor window. */ @@ -53,6 +52,8 @@ export class ClientProject implements Disposable, EngineLibProvider { this.fs = this._store.add( new ProjectFs(initial.id, rpc, caches.contents(initial.id), caches.tree(initial.id)), ); + const lease = caches.opened?.(initial.id); + if (lease) this._store.add(lease); this._store.add( rpc.subscribe(ProjectsContract, 'model', { id: initial.id }, (model) => this._model.set(model), diff --git a/packages/project/src/service/project-service.ts b/packages/project/src/service/project-service.ts index 6a40364f..c675599f 100644 --- a/packages/project/src/service/project-service.ts +++ b/packages/project/src/service/project-service.ts @@ -11,7 +11,7 @@ import { } from '@nanoforge-dev/editor-protocol'; import type { RpcClient } from '@nanoforge-dev/editor-rpc'; -import { type CacheFactory, ClientProject, defaultCacheFactory } from './client-project'; +import { type CacheFactory, ClientProject, memoryCacheFactory } from './client-project'; /** Opens projects through the editor server; one project is active per window. */ export class ProjectService implements Disposable { @@ -19,9 +19,14 @@ export class ProjectService implements Disposable { constructor( private readonly _rpc: RpcClient, - private readonly _caches: CacheFactory = defaultCacheFactory, + private _caches: CacheFactory = memoryCacheFactory(), ) {} + /** Where the next projects opened cache their files (the browser's caches, once known). */ + useCaches(caches: CacheFactory): void { + this._caches = caches; + } + get current(): Observable { return this._current.readonly(); } diff --git a/packages/project/test/cache/content-cache.browser.spec.ts b/packages/project/test/cache/content-cache.browser.spec.ts new file mode 100644 index 00000000..b98e7fcc --- /dev/null +++ b/packages/project/test/cache/content-cache.browser.spec.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; + +import type { ContentCache } from '../../src/cache/cache.type'; +import { MemoryContentCache } from '../../src/cache/memory.content-cache'; +import { writeOpfsFile } from '../../src/cache/opfs-write'; +import { OPFS_LAYOUT, OPFS_ROOT, OpfsContentCache } from '../../src/cache/opfs.content-cache'; +import { sha1Hex } from '../../src/cache/sha1'; + +const encoder = new TextEncoder(); +/** `sha1('héllo, 世界\n')` as the server computes it (Node's crypto). */ +const SERVER_HASH = '67e9b2991ae39a03bb20684bd75feb048c389564'; + +const user = () => `test-${Math.random().toString(36).slice(2)}`; + +describe.each([ + ['memory', () => new MemoryContentCache()], + ['OPFS', () => new OpfsContentCache(user())], +] as const)('%s content cache', (_name, create: () => ContentCache) => { + it('gives back the bytes kept under their hash, nothing for others', async () => { + const cache = create(); + const bytes = encoder.encode('héllo, 世界\n'); + const hash = await sha1Hex(bytes); + expect(hash).toBe(SERVER_HASH); + await cache.set(hash, bytes); + expect(await cache.get(hash)).toEqual(bytes); + expect(await cache.get('0'.repeat(40))).toBeUndefined(); + }); +}); + +describe('OPFS content cache', () => { + it('never gives back bytes that do not match their hash, and drops them', async () => { + const name = user(); + const cache = new OpfsContentCache(name); + const hash = await sha1Hex(encoder.encode('right')); + await writeOpfsFile( + [OPFS_ROOT, OPFS_LAYOUT, encodeURIComponent(name), 'blobs', hash], + encoder.encode('wrong'), + ); + expect(await cache.get(hash)).toBeUndefined(); + await cache.set(hash, encoder.encode('right')); + expect(new TextDecoder().decode(await cache.get(hash))).toBe('right'); + }); + + it('writes through a worker where createWritable is missing', async () => { + const name = user(); + const cache = new OpfsContentCache(name); + const bytes = encoder.encode('written by a worker'); + const hash = await sha1Hex(bytes); + await writeOpfsFile([OPFS_ROOT, OPFS_LAYOUT, encodeURIComponent(name), 'blobs', hash], bytes, { + viaWorker: true, + }); + expect(await cache.get(hash)).toEqual(bytes); + }); + + it('clears what one user kept, not another', async () => { + const [first, second] = [new OpfsContentCache(user()), new OpfsContentCache(user())]; + const bytes = encoder.encode('shared'); + const hash = await sha1Hex(bytes); + await first.set(hash, bytes); + await second.set(hash, bytes); + await first.clear(); + expect(await first.get(hash)).toBeUndefined(); + expect(await second.get(hash)).toEqual(bytes); + }); +}); diff --git a/packages/project/test/cache/project-caches.browser.spec.ts b/packages/project/test/cache/project-caches.browser.spec.ts new file mode 100644 index 00000000..d1b76de5 --- /dev/null +++ b/packages/project/test/cache/project-caches.browser.spec.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from 'vitest'; + +import type { FileEntry } from '@nanoforge-dev/editor-protocol'; + +import { OpfsContentCache } from '../../src/cache/opfs.content-cache'; +import { ProjectCaches } from '../../src/cache/project-caches'; +import { sha1Hex } from '../../src/cache/sha1'; + +const encoder = new TextEncoder(); +const DAY = 24 * 60 * 60 * 1000; + +/** A cache of a fresh user; contents are 1 KiB each, the budget a number of them. */ +const setup = (budgetKiB = 100, maxAgeDays = 30) => { + const user = `test-${Math.random().toString(36).slice(2)}`; + const store = new OpfsContentCache(user); + const caches = new ProjectCaches(user, store, { + budgetBytes: () => budgetKiB * 1024, + maxAgeDays: () => maxAgeDays, + }); + return { user, store, caches }; +}; + +const content = async (text: string) => { + const bytes = encoder.encode(text.padEnd(1024, '.')); + return { bytes, hash: await sha1Hex(bytes) }; +}; + +/** Saves a project's tree holding these contents, as `ProjectFs` does. */ +const tree = async (caches: ProjectCaches, projectId: string, hashes: readonly string[]) => + caches.tree(projectId).save( + hashes.map((hash, index): FileEntry => ({ + path: `f${index}`, + kind: 'file', + size: 1024, + mtime: 0, + hash, + })), + ); + +describe('ProjectCaches', () => { + it('reports what each project holds, counting a shared content once in the total', async () => { + const { caches } = setup(); + const shared = await content('shared'); + const own = await content('own'); + await caches.contents().set(shared.hash, shared.bytes); + await caches.contents().set(own.hash, own.bytes); + caches.opened('a').dispose(); + caches.opened('b').dispose(); + await tree(caches, 'a', [shared.hash, own.hash]); + await tree(caches, 'b', [shared.hash]); + const usage = await caches.usage(); + expect(Object.fromEntries(usage.projects.map((p) => [p.projectId, p.bytes]))).toEqual({ + a: 2048, + b: 1024, + }); + expect(usage.bytes).toBe(2048); + await caches.clearAll(); + }); + + it('drops the contents of a project removed, keeps the ones another project uses', async () => { + const { caches } = setup(); + const shared = await content('shared'); + const own = await content('own'); + for (const item of [shared, own]) await caches.contents().set(item.hash, item.bytes); + caches.opened('a').dispose(); + caches.opened('b').dispose(); + await tree(caches, 'a', [shared.hash, own.hash]); + await tree(caches, 'b', [shared.hash]); + await caches.clearProject('a'); + expect(await caches.contents().get(own.hash)).toBeUndefined(); + expect(await caches.contents().get(shared.hash)).toEqual(shared.bytes); + expect((await caches.usage()).projects.map((p) => p.projectId)).toEqual(['b']); + await caches.clearAll(); + }); + + it('keeps to its budget, oldest first, never the contents of an open project', async () => { + const { caches } = setup(2); + const items = await Promise.all(['one', 'two', 'three'].map(content)); + for (const item of items) await caches.contents().set(item.hash, item.bytes); + const lease = caches.opened('open'); + caches.opened('closed').dispose(); + await tree(caches, 'open', [items[0]!.hash]); + await tree(caches, 'closed', [items[1]!.hash, items[2]!.hash]); + /** Contents written this minute are kept (another tab may be about to use them): age them. */ + await caches.sweep({ recentToo: true }); + const left = await Promise.all(items.map((item) => caches.contents().get(item.hash))); + expect(left.map(Boolean)).toEqual([true, false, true]); + lease.dispose(); + await caches.clearAll(); + }); + + it('forgets projects not opened for a while', async () => { + const { caches } = setup(100, 1); + const item = await content('old'); + await caches.contents().set(item.hash, item.bytes); + caches.opened('old').dispose(); + await tree(caches, 'old', [item.hash]); + const realNow = Date.now; + Date.now = () => realNow() + 2 * DAY; + try { + await caches.sweep({ recentToo: true }); + } finally { + Date.now = realNow; + } + expect((await caches.usage()).projects).toEqual([]); + expect(await caches.contents().get(item.hash)).toBeUndefined(); + await caches.clearAll(); + }); + + it('makes room and writes again when the browser says the quota is exceeded', async () => { + const { user, caches } = setup(2); + const old = await content('old'); + await caches.contents().set(old.hash, old.bytes); + caches.opened('closed').dispose(); + await tree(caches, 'closed', [old.hash]); + const store = (caches as unknown as { _store: OpfsContentCache })._store; + const set = store.set.bind(store); + let refused = false; + store.set = async (hash, bytes) => { + if (!refused) { + refused = true; + throw Object.assign(new Error('full'), { name: 'QuotaExceededError' }); + } + return set(hash, bytes); + }; + const fresh = await content('fresh'); + await caches.contents().set(fresh.hash, fresh.bytes); + expect(refused).toBe(true); + expect(await caches.contents().get(fresh.hash)).toEqual(fresh.bytes); + expect(user).toBeTruthy(); + await caches.clearAll(); + }); +}); diff --git a/packages/project/test/fs/project-fs.spec.ts b/packages/project/test/fs/project-fs.spec.ts index d12b57a2..027c50c6 100644 --- a/packages/project/test/fs/project-fs.spec.ts +++ b/packages/project/test/fs/project-fs.spec.ts @@ -1,14 +1,13 @@ -import { cpSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { cpSync, mkdtempSync, rmSync, statSync, utimesSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import type { FileChange } from '@nanoforge-dev/editor-protocol'; +import { type FileChange, FsContract } from '@nanoforge-dev/editor-protocol'; import type { RpcClient } from '@nanoforge-dev/editor-rpc'; import { type EditorServer, createEditorServer, loadEnv } from '@nanoforge-dev/editor-server-core'; import { createLoopbackClient } from '@nanoforge-dev/editor-server-core/testing'; -import type { ContentCache } from '../../src/cache/cache.type'; import { MemoryContentCache } from '../../src/cache/memory.content-cache'; import { MemoryTreeCache } from '../../src/cache/memory.tree-cache'; import { ProjectService } from '../../src/service/project-service'; @@ -20,18 +19,9 @@ let editor: EditorServer; let rpc: RpcClient; let service: ProjectService; const contents = new MemoryContentCache(); +/** Files the server sent bytes for, and files it answered "not modified" for. */ const reads: string[] = []; - -/** Content cache spy: counts server reads by observing cache fills. */ -const spyCache = (inner: ContentCache): ContentCache => ({ - get: (path) => inner.get(path), - set: (path, value) => { - reads.push(path); - return inner.set(path, value); - }, - delete: (path) => inner.delete(path), - clear: () => inner.clear(), -}); +const notModified: string[] = []; const waitFor = async (predicate: () => boolean, timeout = 2000) => { const start = Date.now(); @@ -50,8 +40,24 @@ beforeAll(async () => { version: 'test', }); rpc = createLoopbackClient(editor); + const api = rpc.api.bind(rpc); + rpc.api = ((contract: Parameters[0]) => { + const methods = api(contract); + if (contract !== FsContract) return methods; + const fs = methods as ReturnType>; + return { + ...fs, + readMany: async (input: Parameters[0]) => { + const answer = await fs.readMany(input); + for (const file of answer.files) + if ('notModified' in file) notModified.push(file.path); + else if (!('error' in file)) reads.push(file.path); + return answer; + }, + }; + }) as RpcClient['api']; service = new ProjectService(rpc, { - contents: () => spyCache(contents), + contents: () => contents, tree: () => new MemoryTreeCache(), }); }); @@ -113,4 +119,39 @@ describe('ProjectFs', () => { code: 'CONFLICT', }); }); + + it('reads again only what changed: not a touch, a rename, or a current copy', async () => { + const project = service.current.get()!; + const path = 'apps/client/src/main.ts'; + const file = join(workspace, 'pong', path); + await project.fs.readText(path); + reads.length = 0; + notModified.length = 0; + + const later = new Date(Date.now() + 120_000); + utimesSync(file, later, later); + await project.fs.refresh(); + expect((await project.fs.readText(path, { fresh: true })).text).toContain(''); + expect(reads).toEqual([]); + expect(notModified).toEqual([path]); + + const before = statSync(file); + const text = (await project.fs.readText(path)).text; + writeFileSync( + file, + text.replace(/./, (char) => (char === 'x' ? 'y' : 'x')), + ); + utimesSync(file, before.atime, before.mtime); + await project.fs.refresh(); + expect((await project.fs.readText(path)).text).not.toBe(text); + expect(reads).toEqual([path]); + + reads.length = 0; + await project.fs.rename(path, 'apps/client/src/renamed.ts'); + await project.fs.readText('apps/client/src/renamed.ts'); + expect(reads).toEqual([]); + await project.fs.rename('apps/client/src/renamed.ts', path); + writeFileSync(file, text); + await project.fs.refresh(); + }); }); diff --git a/packages/project/vitest.config.ts b/packages/project/vitest.config.ts new file mode 100644 index 00000000..2e099572 --- /dev/null +++ b/packages/project/vitest.config.ts @@ -0,0 +1,44 @@ +import { playwright } from '@vitest/browser-playwright'; +import { fileURLToPath } from 'node:url'; +import { defineConfig } from 'vitest/config'; + +const root = (path: string) => fileURLToPath(new URL(`../../${path}`, import.meta.url)); + +/** + * Node tests (`*.spec.ts`) and real browser tests of the caches (`*.browser.spec.ts`: OPFS, + * IndexedDB, Web Locks) on Chromium and Firefox. WebKit needs system libraries some Linux + * distributions lack: `BROWSERS=chromium,firefox,webkit` adds it where it runs. + */ +export default defineConfig({ + test: { + passWithNoTests: true, + setupFiles: [root('vitest.setup.ts')], + projects: [ + { + extends: true, + test: { + name: 'node', + environment: 'node', + include: ['test/**/*.spec.ts'], + exclude: ['**/*.browser.spec.ts'], + globalSetup: [root('vitest.global-setup.ts')], + }, + }, + { + extends: true, + test: { + name: 'browser', + include: ['test/**/*.browser.spec.ts'], + browser: { + enabled: true, + headless: true, + provider: playwright(), + instances: (process.env.BROWSERS ?? 'chromium,firefox').split(',').map((browser) => ({ + browser: browser.trim() as 'chromium' | 'firefox' | 'webkit', + })), + }, + }, + }, + ], + }, +}); diff --git a/packages/protocol/src/fs/fs.contract.ts b/packages/protocol/src/fs/fs.contract.ts index ffa94ebf..43c76343 100644 --- a/packages/protocol/src/fs/fs.contract.ts +++ b/packages/protocol/src/fs/fs.contract.ts @@ -4,7 +4,7 @@ import { defineContract } from '@nanoforge-dev/editor-rpc'; import { ProjectPath } from '../path/project-path'; import { ProjectId } from '../project/project.schema'; -import { Bytes, FileChange, FileContent, FileEntry } from './fs.schema'; +import { Bytes, FileChange, FileContent, FileEntry, ReadRequest, ReadResult } from './fs.schema'; const Target = z.object({ project: ProjectId, path: ProjectPath }); @@ -16,6 +16,15 @@ export const FsContract = defineContract('fs', { output: z.array(FileEntry), }, read: { input: Target, output: FileContent }, + /** + * Several files in one call (a project opening, its open tabs). A file whose `knownHash` is + * current comes back `notModified`, without bytes. Answers stop past a few MiB: `rest` holds + * the paths still to ask for. + */ + readMany: { + input: z.object({ project: ProjectId, files: z.array(ReadRequest).max(1000) }), + output: z.object({ files: z.array(ReadResult), rest: z.array(ProjectPath) }), + }, write: { input: Target.extend({ content: z.union([Bytes, z.string()]), diff --git a/packages/protocol/src/fs/fs.schema.ts b/packages/protocol/src/fs/fs.schema.ts index 3d2df3d7..e35f1238 100644 --- a/packages/protocol/src/fs/fs.schema.ts +++ b/packages/protocol/src/fs/fs.schema.ts @@ -13,6 +13,11 @@ export const FileEntry = z.object({ size: z.number().int().nonnegative(), /** Modification time in ms since epoch. */ mtime: z.number(), + /** + * sha1 of a file's content (files the server can read: 64 MiB at most). A client trusts its + * cached copy of the content when the hashes match, whatever the modification time says. + */ + hash: z.string().optional(), }); export type FileEntry = z.output; @@ -29,3 +34,21 @@ export const FileChange = z.object({ kind: FileKind, }); export type FileChange = z.output; + +/** One file asked for by `readMany`, with the hash of the copy the client already has. */ +export const ReadRequest = z.object({ path: ProjectPath, knownHash: z.string().optional() }); +export type ReadRequest = z.output; + +/** + * One answer of `readMany`: the content, or `notModified` when the client's copy is current + * (no bytes sent), or why the file could not be read. + */ +export const ReadResult = z.union([ + FileContent, + FileEntry.extend({ notModified: z.literal(true), hash: z.string() }), + z.object({ + path: ProjectPath, + error: z.object({ code: z.string(), message: z.string() }), + }), +]); +export type ReadResult = z.output; diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 786d3309..7a3aa143 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -80,10 +80,14 @@ export { } from '@nanoforge-dev/editor-kernel'; export { + type CacheUsage, type ClientProject, + type ProjectCaches, + ProjectCachesToken, type ProjectFs, type ProjectService, ProjectServiceToken, + type ProjectUsage, type WriteOptions, } from '@nanoforge-dev/editor-project'; export type { diff --git a/packages/sdk/test/index.spec.ts b/packages/sdk/test/index.spec.ts index 888adb43..7e7acb79 100644 --- a/packages/sdk/test/index.spec.ts +++ b/packages/sdk/test/index.spec.ts @@ -55,6 +55,7 @@ describe('public API', () => { "MutableDisposable", "ObservableValue", "PluginHostToken", + "ProjectCachesToken", "ProjectServiceToken", "REQUIRED_PLUGINS", "RUNTIME_PROTOCOL_VERSION", diff --git a/packages/server-core/src/fs/file-hashes.ts b/packages/server-core/src/fs/file-hashes.ts new file mode 100644 index 00000000..a1a0ca0d --- /dev/null +++ b/packages/server-core/src/fs/file-hashes.ts @@ -0,0 +1,116 @@ +import { createHash } from 'node:crypto'; +import { createReadStream } from 'node:fs'; +import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; + +/** Hashes kept at most; the oldest are forgotten first. */ +const MAX_ENTRIES = 50_000; +/** Saving waits this long after the last new hash. */ +const SAVE_DELAY_MS = 2000; + +/** What tells a file's versions apart without reading it. */ +export interface FileVersion { + readonly size: number; + readonly mtimeMs: number; + /** Changed by every write, and unlike mtime, never set back by a program. */ + readonly ctimeMs: number; + readonly ino: number; +} + +export interface FileHashesOptions { + /** Files larger than this get no hash (they are never read whole). */ + readonly maxBytes: number; + /** + * Where the hashes are kept between runs (`DATA_DIR/file-hashes.json`): without it, the first + * listing after a start reads every file of the project again. + */ + readonly file?: string; +} + +/** + * sha1 of files, kept by path and version (size, mtime, ctime, inode): a listing hashes each + * file once, and again only after it changed. Files are read as streams, so memory stays small + * whatever their size. Clients compare the hash with the copy they cached. + */ +export class FileHashes { + private readonly _hashes = new Map(); + private _loaded: Promise | undefined; + private _saveTimer: ReturnType | undefined; + + constructor(private readonly _options: FileHashesOptions) {} + + /** The hash of a file, or undefined when it is too large to be read (and cached) at all. */ + async of(absolute: string, info: FileVersion): Promise { + if (info.size > this._options.maxBytes) return undefined; + await this._load(); + const key = keyOf(absolute, info); + const known = this._hashes.get(key); + if (known) return known; + const hash = await streamSha1(absolute); + this.remember(absolute, info, hash); + return hash; + } + + /** Records a hash computed elsewhere (a read or a write of the file). */ + remember(absolute: string, info: FileVersion, hash: string): void { + const key = keyOf(absolute, info); + this._hashes.delete(key); + this._hashes.set(key, hash); + if (this._hashes.size > MAX_ENTRIES) + this._hashes.delete(this._hashes.keys().next().value as string); + this._scheduleSave(); + } + + /** Writes what is waiting to be saved. */ + async flush(): Promise { + if (!this._saveTimer) return; + clearTimeout(this._saveTimer); + this._saveTimer = undefined; + await this._save(); + } + + private _load(): Promise { + const file = this._options.file; + this._loaded ??= file + ? readFile(file, 'utf8') + .then((text) => { + for (const [key, hash] of JSON.parse(text) as [string, string][]) + if (!this._hashes.has(key)) this._hashes.set(key, hash); + }) + .catch(() => undefined) + : Promise.resolve(); + return this._loaded; + } + + private _scheduleSave(): void { + if (!this._options.file || this._saveTimer) return; + this._saveTimer = setTimeout(() => { + this._saveTimer = undefined; + void this._save(); + }, SAVE_DELAY_MS); + this._saveTimer.unref?.(); + } + + private async _save(): Promise { + const file = this._options.file; + if (!file) return; + try { + await mkdir(dirname(file), { recursive: true }); + const temp = `${file}.${process.pid}.tmp`; + await writeFile(temp, JSON.stringify([...this._hashes])); + await rename(temp, file); + } catch {} + } +} + +const keyOf = (absolute: string, info: FileVersion) => + `${absolute}\0${info.size}\0${info.mtimeMs}\0${info.ctimeMs}\0${info.ino}`; + +const streamSha1 = (absolute: string): Promise => + new Promise((resolve, reject) => { + const hash = createHash('sha1'); + createReadStream(absolute) + .on('data', (chunk) => hash.update(chunk)) + .on('end', () => resolve(hash.digest('hex'))) + .on('error', reject); + }); diff --git a/packages/server-core/src/fs/fs-service.ts b/packages/server-core/src/fs/fs-service.ts index 7fa3b1e2..f3827fac 100644 --- a/packages/server-core/src/fs/fs-service.ts +++ b/packages/server-core/src/fs/fs-service.ts @@ -4,11 +4,17 @@ import { existsSync } from 'node:fs'; import { cp, mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises'; import { basename, dirname, join } from 'node:path'; -import type { FileContent, FileEntry } from '@nanoforge-dev/editor-protocol'; +import type { + FileContent, + FileEntry, + ReadRequest, + ReadResult, +} from '@nanoforge-dev/editor-protocol'; import { RpcError } from '@nanoforge-dev/editor-rpc'; import { sha1 } from '../util/hash'; import { KeyedMutex } from '../util/keyed-mutex'; +import { FileHashes, type FileVersion } from './file-hashes'; import { PathJail } from './path-jail'; export const EDITOR_DIR = '.nanoforge/editor'; @@ -19,7 +25,9 @@ export const NFIGNORE_FILE = '.nfignore'; export const PACKAGES_DIR = 'nf_modules'; const EDITOR_GITIGNORE = 'trash/\nlocal.json\n'; -const MAX_READ_BYTES = 64 * 1024 * 1024; +export const MAX_READ_BYTES = 64 * 1024 * 1024; +/** Bytes `readMany` sends in one answer (a file larger than this still goes, alone). */ +const MAX_BATCH_BYTES = 8 * 1024 * 1024; const notFound = (path: string) => new RpcError('NOT_FOUND', `No such file or directory: ${path}`); @@ -73,9 +81,15 @@ const assertWritable = (path: string) => { export class ProjectFileSystem { readonly jail: PathJail; private readonly _writes = new KeyedMutex(); + private readonly _hashes: FileHashes; private _ignore: { mtime: number; rules: Ignore } | undefined; - constructor(readonly root: string) { + constructor( + readonly root: string, + /** Hashes shared by the server's projects (kept between runs); its own otherwise. */ + hashes?: FileHashes, + ) { + this._hashes = hashes ?? new FileHashes({ maxBytes: MAX_READ_BYTES }); this.jail = new PathJail(root, gitReason); } @@ -105,12 +119,9 @@ export class ProjectFileSystem { const absolute = await this.jail.resolve(path); try { const info = await stat(absolute); - return { - path, - kind: info.isDirectory() ? 'directory' : 'file', - size: info.size, - mtime: info.mtimeMs, - }; + const kind = info.isDirectory() ? 'directory' : 'file'; + const hash = kind === 'file' ? await this._hashes.of(absolute, info) : undefined; + return { path, kind, size: info.size, mtime: info.mtimeMs, ...(hash && { hash }) }; } catch (error) { if (errno(error) === 'ENOENT') return null; return mapError(error, path); @@ -122,6 +133,8 @@ export class ProjectFileSystem { const absolute = await this.jail.resolve(path); await this.refreshIgnore(); const entries: FileEntry[] = []; + /** Files to hash once the walk is done, a few at a time. */ + const files: { index: number; absolute: string; info: FileVersion }[] = []; const walk = async (dir: string, rel: string) => { let names; try { @@ -136,6 +149,8 @@ export class ProjectFileSystem { if (!info) continue; const kind = info.isDirectory() ? 'directory' : 'file'; if (this.isIgnored(childRel, kind === 'directory')) continue; + if (kind === 'file') + files.push({ index: entries.length, absolute: join(dir, entry.name), info }); entries.push({ path: childRel, kind, size: info.size, mtime: info.mtimeMs }); if (recursive && kind === 'directory' && !SKIPPED_IN_RECURSIVE_LIST.has(entry.name)) { await walk(join(dir, entry.name), childRel); @@ -143,6 +158,14 @@ export class ProjectFileSystem { } }; await walk(absolute, path); + const queue = [...files]; + const hashNext = async () => { + for (let file = queue.shift(); file; file = queue.shift()) { + const hash = await this._hashes.of(file.absolute, file.info).catch(() => undefined); + if (hash) entries[file.index] = { ...entries[file.index]!, hash }; + } + }; + await Promise.all(Array.from({ length: Math.min(8, queue.length) }, hashNext)); return entries; } @@ -155,20 +178,50 @@ export class ProjectFileSystem { if (info.size > MAX_READ_BYTES) throw new RpcError('PAYLOAD_TOO_LARGE', `File too large: ${path}`); const content = new Uint8Array(await readFile(absolute)); - return { - path, - kind: 'file', - size: info.size, - mtime: info.mtimeMs, - content, - hash: sha1(content), - }; + const hash = sha1(content); + this._hashes.remember(absolute, info, hash); + return { path, kind: 'file', size: info.size, mtime: info.mtimeMs, content, hash }; } catch (error) { if (error instanceof RpcError) throw error; return mapError(error, path); } } + /** + * Several files at once: a file whose `knownHash` is current comes back `notModified`, without + * its bytes. Answers stop past {@link MAX_BATCH_BYTES}; `rest` holds the paths not read yet. + */ + async readMany(files: readonly ReadRequest[]): Promise<{ files: ReadResult[]; rest: string[] }> { + const results: ReadResult[] = []; + let bytes = 0; + /** Eight files at a time, as separate reads would run; sizes are known before reading. */ + for (let start = 0; start < files.length;) { + const window = files.slice(start, start + 8); + const sizes = await Promise.all( + window.map(async (file) => { + try { + return (await stat(await this.jail.resolve(file.path))).size; + } catch { + return 0; + } + }), + ); + let take = 0; + for (const size of sizes) { + /** A file that would cross the cap waits for the next answer, unless it comes alone. */ + if (bytes + size > MAX_BATCH_BYTES && (results.length || take)) break; + bytes += size; + take++; + } + const batch = await Promise.all(window.slice(0, take).map((file) => this._readOne(file))); + results.push(...batch); + start += take; + if (take < window.length) + return { files: results, rest: files.slice(start).map((rest) => rest.path) }; + } + return { files: results, rest: [] }; + } + /** * Atomic write (temp file + rename). With `expectedHash`, fails with CONFLICT when the file * changed since it was read (`null` means "must not exist"). @@ -204,7 +257,9 @@ export class ProjectFileSystem { return mapError(error, path); } const info = await stat(absolute); - return { path, kind: 'file', size: info.size, mtime: info.mtimeMs, hash: sha1(content) }; + const hash = sha1(content); + this._hashes.remember(absolute, info, hash); + return { path, kind: 'file', size: info.size, mtime: info.mtimeMs, hash }; }); } @@ -279,6 +334,24 @@ export class ProjectFileSystem { return (await this.stat(to))!; } + private async _readOne(file: ReadRequest): Promise { + try { + const entry = file.knownHash ? await this.stat(file.path) : undefined; + if (entry === null) throw notFound(file.path); + if (entry?.hash && entry.hash === file.knownHash) + return { ...entry, hash: entry.hash, notModified: true }; + return await this.read(file.path); + } catch (error) { + return { + path: file.path, + error: { + code: error instanceof RpcError ? error.code : 'INTERNAL', + message: error instanceof Error ? error.message : String(error), + }, + }; + } + } + private async _ensureEditorGitignore(): Promise { const file = await this.jail.resolve(`${EDITOR_DIR}/.gitignore`); if (!existsSync(file)) await writeFile(file, EDITOR_GITIGNORE); diff --git a/packages/server-core/src/project/project-registry.ts b/packages/server-core/src/project/project-registry.ts index c23074ef..e753dca3 100644 --- a/packages/server-core/src/project/project-registry.ts +++ b/packages/server-core/src/project/project-registry.ts @@ -17,7 +17,8 @@ import { RpcError } from '@nanoforge-dev/editor-rpc'; import type { ApiClient } from '../api/api-client'; import type { EditorEnv } from '../env/editor-env.type'; -import { NFIGNORE_FILE, ProjectFileSystem } from '../fs/fs-service'; +import { FileHashes } from '../fs/file-hashes'; +import { MAX_READ_BYTES, NFIGNORE_FILE, ProjectFileSystem } from '../fs/fs-service'; import { PathJail } from '../fs/path-jail'; import { ProjectWatcher } from '../fs/watcher'; import type { GitService } from '../git/git-service'; @@ -53,8 +54,9 @@ export class OpenProject implements Disposable { initial: ProjectModel, private readonly _loader: ConfigLoader, private readonly _logger: Logger, + hashes?: FileHashes, ) { - this.fs = new ProjectFileSystem(root); + this.fs = new ProjectFileSystem(root, hashes); this._model = new ObservableValue(initial); this.watcher = new ProjectWatcher(this.fs.jail, { ignoredPaths: () => this._model.get().apps.flatMap((app) => (app.outDir ? [app.outDir] : [])), @@ -118,6 +120,8 @@ export class ProjectRegistry implements Disposable { private readonly _lastUsed = new Map(); private readonly _opening = new KeyedMutex(); private readonly _recentLock = new KeyedMutex(); + /** File hashes of every project, kept between runs (`DATA_DIR/file-hashes.json`). */ + private readonly _hashes: FileHashes; private readonly _onDidClose = new Emitter(); /** The folders whose code a local editor may run. */ readonly trust: TrustStore; @@ -131,6 +135,10 @@ export class ProjectRegistry implements Disposable { private readonly _api: ApiClient, private readonly _logger: Logger, ) { + this._hashes = new FileHashes({ + maxBytes: MAX_READ_BYTES, + file: join(_env.dataDir, 'file-hashes.json'), + }); this.trust = new TrustStore(_env.dataDir, async () => Object.values(await this._readRecent()).flatMap((entries) => entries.flatMap((entry) => ('path' in entry.ref ? [entry.ref.path] : [])), @@ -220,6 +228,7 @@ export class ProjectRegistry implements Disposable { } dispose(): void { + void this._hashes.flush(); for (const project of this._projects.values()) project.dispose(); this._projects.clear(); this._onDidClose.dispose(); @@ -286,6 +295,7 @@ export class ProjectRegistry implements Disposable { model, loader, this._logger, + this._hashes, ); this._projects.set(id, project); this._logger.info(`Opened project ${model.name} (${id}) at ${root}`); diff --git a/packages/server-core/src/rpc/implementation/fs.implementation.ts b/packages/server-core/src/rpc/implementation/fs.implementation.ts index 37cc1336..1318cd47 100644 --- a/packages/server-core/src/rpc/implementation/fs.implementation.ts +++ b/packages/server-core/src/rpc/implementation/fs.implementation.ts @@ -18,6 +18,8 @@ export const implementFsRpc = ( list: ({ project, path, recursive }, { session }) => projects.get(session, project).fs.list(path, recursive), read: ({ project, path }, { session }) => projects.get(session, project).fs.read(path), + readMany: ({ project, files }, { session }) => + projects.get(session, project).fs.readMany(files), write: ({ project, path, content, expectedHash, createParents }, { session }) => projects.get(session, project).fs.write(path, content, { ...(expectedHash !== undefined && { expectedHash }), diff --git a/packages/server-core/src/server/create-editor-server.ts b/packages/server-core/src/server/create-editor-server.ts index 2f9d548d..1dadfa7b 100644 --- a/packages/server-core/src/server/create-editor-server.ts +++ b/packages/server-core/src/server/create-editor-server.ts @@ -228,8 +228,16 @@ export const createEditorServer = (options: EditorServerOptions): EditorServer = }, }); } + /** The content hash is the ETag: a thumbnail asked again with it comes back empty. */ + const etag = entry.hash ? `"${entry.hash}"` : undefined; + if (etag && !download && request.headers.get('if-none-match') === etag) + return new Response(null, { + status: 304, + headers: { etag, 'cache-control': 'no-cache', 'content-security-policy': 'sandbox' }, + }); const response = await fileResponse(await project.fs.jail.resolve(path), 'no-cache'); if (!response) return json(404, { error: 'Not found' }); + if (etag) response.headers.set('etag', etag); response.headers.set('content-security-policy', 'sandbox'); if (download) response.headers.set('content-disposition', disposition(name)); return response; diff --git a/packages/server-core/test/fs/file-hashes.spec.ts b/packages/server-core/test/fs/file-hashes.spec.ts new file mode 100644 index 00000000..4bb7955b --- /dev/null +++ b/packages/server-core/test/fs/file-hashes.spec.ts @@ -0,0 +1,32 @@ +import { mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, describe, expect, it } from 'vitest'; + +import { FileHashes } from '../../src/fs/file-hashes'; +import { sha1 } from '../../src/util/hash'; + +const dir = mkdtempSync(join(tmpdir(), 'nf-hashes-')); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +describe('FileHashes', () => { + it('keeps hashes between runs: a restarted server reads nothing again', async () => { + const file = join(dir, 'asset.bin'); + writeFileSync(file, 'content'); + const info = statSync(file); + const store = join(dir, 'data/file-hashes.json'); + const first = new FileHashes({ maxBytes: 1024, file: store }); + expect(await first.of(file, info)).toBe(sha1('content')); + await first.flush(); + + rmSync(file); + const second = new FileHashes({ maxBytes: 1024, file: store }); + expect(await second.of(file, info)).toBe(sha1('content')); + }); + + it('gives no hash to a file over the limit', async () => { + const file = join(dir, 'big.bin'); + writeFileSync(file, 'x'.repeat(2048)); + expect(await new FileHashes({ maxBytes: 1024 }).of(file, statSync(file))).toBeUndefined(); + }); +}); diff --git a/packages/server-core/test/fs/fs-service.spec.ts b/packages/server-core/test/fs/fs-service.spec.ts index 32d36a43..087320de 100644 --- a/packages/server-core/test/fs/fs-service.spec.ts +++ b/packages/server-core/test/fs/fs-service.spec.ts @@ -6,6 +6,7 @@ import { readdirSync, rmSync, symlinkSync, + utimesSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -31,6 +32,34 @@ afterEach(() => { }); describe('ProjectFileSystem', () => { + it('gives files their content hash, the same after a touch, another after an edit', async () => { + const { root, fs } = project(); + const listed = await fs.list('', true); + expect(listed.find((entry) => entry.path === 'src')?.hash).toBeUndefined(); + expect(listed.find((entry) => entry.path === 'src/main.ts')?.hash).toBe(sha1('export {};\n')); + const later = new Date(Date.now() + 60_000); + utimesSync(join(root, 'src/main.ts'), later, later); + expect((await fs.stat('src/main.ts'))?.hash).toBe(sha1('export {};\n')); + writeFileSync(join(root, 'src/main.ts'), 'export [];\n'); + utimesSync(join(root, 'src/main.ts'), later, later); + expect((await fs.stat('src/main.ts'))?.hash).toBe(sha1('export [];\n')); + }); + + it('answers readMany within 8 MiB, the rest for the next call', async () => { + const { root, fs } = project(); + const big = new Uint8Array(5 * 1024 * 1024); + for (const name of ['a', 'b', 'c']) writeFileSync(join(root, `src/${name}.bin`), big); + const first = await fs.readMany(['a', 'b', 'c'].map((name) => ({ path: `src/${name}.bin` }))); + expect(first.files.map((file) => file.path)).toEqual(['src/a.bin']); + expect(first.rest).toEqual(['src/b.bin', 'src/c.bin']); + const known = (await fs.stat('src/a.bin'))!.hash!; + const again = await fs.readMany([{ path: 'src/a.bin', knownHash: known }, { path: 'nope' }]); + expect(again.files).toMatchObject([ + { path: 'src/a.bin', notModified: true }, + { path: 'nope', error: { code: 'NOT_FOUND' } }, + ]); + }); + it('reads, stats and lists', async () => { const { fs } = project(); const file = await fs.read('src/main.ts'); diff --git a/packages/server-core/test/server/create-editor-server.spec.ts b/packages/server-core/test/server/create-editor-server.spec.ts index c41ede91..a885eb27 100644 --- a/packages/server-core/test/server/create-editor-server.spec.ts +++ b/packages/server-core/test/server/create-editor-server.spec.ts @@ -250,14 +250,19 @@ describe('editor server', () => { it('serves project files and downloads folders as zip', async () => { const { id } = await client.api(ProjectsContract).open({ path: 'pong' }); - const fetchFile = async (path: string) => { + const fetchFile = async (path: string, headers: Record = {}) => { const response = await editor.fetch( - new Request(`${ORIGIN}/files/${id}/${path}`, { headers: { cookie: '' } }), + new Request(`${ORIGIN}/files/${id}/${path}`, { headers: { cookie: '', ...headers } }), ); return response!; }; const file = await fetchFile('apps/client/package.json'); expect(file.status).toBe(200); + const etag = file.headers.get('etag'); + expect(etag).toMatch(/^"[0-9a-f]{40}"$/); + const again = await fetchFile('apps/client/package.json', { 'if-none-match': etag! }); + expect(again.status).toBe(304); + expect(await again.text()).toBe(''); expect(file.headers.get('content-security-policy')).toBe('sandbox'); expect(file.headers.get('x-content-type-options')).toBe('nosniff'); expect(await file.json()).toMatchObject({ name: expect.any(String) }); diff --git a/packages/settings/src/definition/core-settings.ts b/packages/settings/src/definition/core-settings.ts index 53638b6b..664cd065 100644 --- a/packages/settings/src/definition/core-settings.ts +++ b/packages/settings/src/definition/core-settings.ts @@ -22,6 +22,26 @@ export const CoreSettings = { category: 'Editor/Appearance', scopes: ['account', 'machine'], }), + cacheBudget: defineSetting({ + key: 'storage.cacheBudgetMb', + schema: z.number().int().min(50).max(100_000), + default: 500, + title: 'Cache size', + description: + 'Space (MB) the browser may use to keep project files, so they need not be downloaded again. Never more than a tenth of what the browser allows. Least recently used files go first.', + category: 'Editor/Storage', + scopes: ['machine'], + }), + cacheMaxAge: defineSetting({ + key: 'storage.cacheMaxAgeDays', + schema: z.number().int().min(0).max(3650), + default: 30, + title: 'Forget projects after (days)', + description: + 'Cached files of a project not opened for this many days are dropped (0 keeps them).', + category: 'Editor/Storage', + scopes: ['machine'], + }), historyLimit: defineSetting({ key: 'history.limit', schema: z.number().int().min(1).max(10_000), diff --git a/plugins/code-editor/src/typing/type-libraries.ts b/plugins/code-editor/src/typing/type-libraries.ts index 6681a0d6..0853404c 100644 --- a/plugins/code-editor/src/typing/type-libraries.ts +++ b/plugins/code-editor/src/typing/type-libraries.ts @@ -95,6 +95,8 @@ export class TypeLibraries implements Disposable { /** The project's code files, a few reads at a time: a big project sent them all at once. */ private async _loadAll(paths: readonly string[]): Promise { + /** Fills the file cache in a few requests; the reads below then cost nothing. */ + await this._project.fs.readMany(paths).catch(() => undefined); const queue = [...paths]; const worker = async () => { for (let path = queue.shift(); path !== undefined && !this._disposed; path = queue.shift()) diff --git a/plugins/settings-ui/src/dialogs/settings/SettingsDialog.svelte b/plugins/settings-ui/src/dialogs/settings/SettingsDialog.svelte index aafac9be..48c0f7d5 100644 --- a/plugins/settings-ui/src/dialogs/settings/SettingsDialog.svelte +++ b/plugins/settings-ui/src/dialogs/settings/SettingsDialog.svelte @@ -27,6 +27,7 @@ import ConflictsPage from './ConflictsPage.svelte'; import ImportExportPage from './ImportExportPage.svelte'; + import StoragePanel from './StoragePanel.svelte'; import PluginsPage from './plugins/PluginsPage.svelte'; import SettingRow from './SettingRow.svelte'; import { categoryTree, searchSettings, settingsIn } from '../../settings/setting-categories'; @@ -264,6 +265,7 @@ onresolve={(key, value) => void account?.resolveConflict(key, value)} /> {:else} + {#if page === 'Editor/Storage'}{/if} {#each shown as definition (definition.key)} + import { + type CacheUsage, + ProjectCachesToken, + ProjectServiceToken, + type ServiceAccessor, + } from '@nanoforge-dev/editor-sdk'; + import { Button, Switch } from '@nanoforge-dev/editor-sdk/ui'; + + /** + * What the browser keeps of the user's projects (Settings › Editor › Storage), above the + * budget and expiry settings: the space each project takes, and ways to clear it. + */ + const { services }: { services: ServiceAccessor } = $props(); + // svelte-ignore state_referenced_locally + const caches = services.tryGet(ProjectCachesToken); + // svelte-ignore state_referenced_locally + const projects = services.tryGet(ProjectServiceToken); + + let usage = $state(); + let names = $state>({}); + let busy = $state(false); + let failed = $state(); + + const load = async () => { + if (!caches) return; + usage = await caches.usage(); + const recent = await projects?.recent().catch(() => []); + names = Object.fromEntries((recent ?? []).map(({ id, name }) => [id, name])); + }; + $effect(() => void load()); + + const run = async (task: () => Promise) => { + busy = true; + failed = undefined; + try { + await task(); + await load(); + } catch (error) { + failed = error instanceof Error ? error.message : String(error); + } finally { + busy = false; + } + }; + + const size = (bytes: number) => + bytes < 1024 * 1024 + ? `${Math.ceil(bytes / 1024)} KB` + : `${(bytes / (1024 * 1024)).toFixed(bytes < 10 * 1024 * 1024 ? 1 : 0)} MB`; + const date = (time: number) => + new Intl.DateTimeFormat(undefined, { dateStyle: 'medium' }).format(new Date(time)); + const canPersist = typeof navigator !== 'undefined' && !!navigator.storage?.persist; + + +
+

Files kept in this browser

+ {#if !caches} +

This browser can't keep project files: they are downloaded each time.

+ {:else if !usage} +

Measuring…

+ {:else} +

+ Project files are kept here so they need not be downloaded again: {size(usage.bytes)} of + {size(usage.budget)}. A file several projects have is kept once. +

+ {#if usage.projects.length} +
    + {#each [...usage.projects].sort((a, b) => b.bytes - a.bytes) as project (project.projectId)} + {@const name = names[project.projectId] ?? project.projectId} +
  • + {name} + {size(project.bytes)} · {project.files} files · opened {date(project.opened)} + +
    • + {/each} +
+ {/if} +
+ + {#if canPersist} + on && void run(() => navigator.storage.persist())} + /> + {#if usage.persisted}Kept: the browser will not remove them on its own.{:else}Keep them when the browser runs short of space{/if} + {/if} +
+ {#if failed}{/if} + {/if} +
+ + diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce711722..ba24ef45 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -583,6 +583,12 @@ importers: '@nanoforge-dev/editor-server-core': specifier: workspace:* version: link:../server-core + '@vitest/browser-playwright': + specifier: catalog:test + version: 4.1.11(playwright@1.63.0)(vite@8.3.1(@types/node@26.6.2)(jiti@2.6.1)(yaml@2.9.1))(vitest@4.1.11) + playwright: + specifier: catalog:test + version: 1.63.0 typescript: specifier: catalog:build version: 6.0.3 From d8bf68899662f2fcbb3da2be98a9f69832f774d7 Mon Sep 17 00:00:00 2001 From: Exelo Date: Fri, 30 Oct 2026 21:34:13 +0100 Subject: [PATCH 2/2] fix: address the cache review - a reply that left before a save, a delete or a server change never puts back an older hash (per-path change clock) - a file the server changed is read from the server until its stat answers - stat and list hash regular files only, and a failed hash is no hash - file hashes are kept one per path, least recently used evicted first, saved only when they change; the trash is not hashed - a save does not wait for the content cache - one shared cache database connection, one sweep for a full quota, records written before files, trees read in one transaction --- .github/workflows/tests.yml | 4 +- packages/project/src/cache/cache-db.ts | 57 ++- packages/project/src/cache/project-caches.ts | 74 +++- packages/project/src/fs/project-fs.ts | 136 +++++-- .../test/cache/project-caches.browser.spec.ts | 35 ++ .../project/test/fs/project-fs-races.spec.ts | 371 ++++++++++++++++++ packages/project/test/fs/project-fs.spec.ts | 3 +- packages/server-core/src/fs/file-hashes.ts | 48 ++- packages/server-core/src/fs/fs-service.ts | 12 +- .../server-core/test/fs/file-hashes.spec.ts | 34 +- .../server-core/test/fs/fs-service.spec.ts | 43 ++ .../test/typing/type-libraries.spec.ts | 3 + 12 files changed, 749 insertions(+), 71 deletions(-) create mode 100644 packages/project/test/fs/project-fs-races.spec.ts diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 078ebcdb..08697043 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,9 +35,9 @@ jobs: run: pnpm typecheck - name: Install browsers for component and cache tests - run: pnpm --filter @nanoforge-dev/editor-ui exec playwright install --with-deps chromium firefox webkit + run: pnpm --filter @nanoforge-dev/editor-ui exec playwright install --with-deps chromium firefox - name: Run tests run: pnpm test:unit env: - BROWSERS: chromium,firefox,webkit + BROWSERS: chromium,firefox diff --git a/packages/project/src/cache/cache-db.ts b/packages/project/src/cache/cache-db.ts index 1a4d8c01..7dc5ec08 100644 --- a/packages/project/src/cache/cache-db.ts +++ b/packages/project/src/cache/cache-db.ts @@ -47,25 +47,60 @@ export const openCacheDb = (): Promise => request.onblocked = () => reject(new Error('IndexedDB upgrade blocked by another tab')); }); +/** The open connection, shared by every transaction of this page. */ +let connection: Promise | undefined; + +/** + * The shared connection, opened on first use. It closes (and is opened again on next use) when + * another tab upgrades the database, and a failed open is not kept. + */ +const connect = (): Promise => { + if (connection) return connection; + const opening = openCacheDb().then( + (db) => { + const forget = () => { + if (connection === opening) connection = undefined; + }; + db.onversionchange = () => { + db.close(); + forget(); + }; + db.onclose = forget; + return db; + }, + (error: unknown) => { + if (connection === opening) connection = undefined; + throw error; + }, + ); + connection = opening; + return opening; +}; + /** Runs requests in one transaction; resolves with the last request's result. */ export const transact = async ( stores: string | string[], mode: IDBTransactionMode, run: (transaction: IDBTransaction) => IDBRequest | IDBRequest[] | undefined, ): Promise => { - const db = await openCacheDb(); + let db = await connect(); + let transaction: IDBTransaction; try { - return await new Promise((resolve, reject) => { - const transaction = db.transaction(stores, mode); - const requests = run(transaction); - const last = Array.isArray(requests) ? requests.at(-1) : requests; - transaction.oncomplete = () => resolve(last?.result as T); - transaction.onerror = () => reject(transaction.error ?? new Error('IndexedDB failed')); - transaction.onabort = () => reject(transaction.error ?? new Error('IndexedDB aborted')); - }); - } finally { - db.close(); + transaction = db.transaction(stores, mode); + } catch (error) { + /** Closed under us (another tab upgraded it): opened again once. */ + if ((error as { name?: string })?.name !== 'InvalidStateError') throw error; + connection = undefined; + db = await connect(); + transaction = db.transaction(stores, mode); } + return new Promise((resolve, reject) => { + const requests = run(transaction); + const last = Array.isArray(requests) ? requests.at(-1) : requests; + transaction.oncomplete = () => resolve(last?.result as T); + transaction.onerror = () => reject(transaction.error ?? new Error('IndexedDB failed')); + transaction.onabort = () => reject(transaction.error ?? new Error('IndexedDB aborted')); + }); }; /** Every value of a store whose key starts with `prefix`. */ diff --git a/packages/project/src/cache/project-caches.ts b/packages/project/src/cache/project-caches.ts index 5a870ab5..c937b75a 100644 --- a/packages/project/src/cache/project-caches.ts +++ b/packages/project/src/cache/project-caches.ts @@ -64,6 +64,8 @@ export class ProjectCaches implements CacheFactory { private _writtenSinceSweep = 0; private _sweepTimer: ReturnType | undefined; private _touchTimer: ReturnType | undefined; + /** The sweep making room after a full disk: every write waiting for room shares it. */ + private _roomSweep: Promise | undefined; private readonly _contents: ContentCache; constructor( @@ -245,31 +247,57 @@ export class ProjectCaches implements CacheFactory { void this._flushTouches(); } + /** + * The record is written before the file: a tab closed in between leaves a record whose file is + * missing (a miss, swept later), never a file no sweep knows of. + */ private async _set(hash: string, content: Uint8Array): Promise { - try { - await this._store.set(hash, content); - } catch (error) { - if ((error as { name?: string })?.name !== 'QuotaExceededError') return; - await this.sweep({ room: content.length }).catch(() => undefined); - try { - await this._store.set(hash, content); - } catch { - return; - } - } - await transact(BLOBS, 'readwrite', (transaction) => + const recorded = await transact(BLOBS, 'readwrite', (transaction) => transaction .objectStore(BLOBS) .put( { user: this.user, hash, size: content.length, used: Date.now() } satisfies BlobRecord, blobKey(this.user, hash), ), - ).catch(() => undefined); + ).then( + () => true, + () => false, + ); + if (!recorded) return; + try { + await this._store.set(hash, content); + } catch (error) { + const stored = + (error as { name?: string })?.name === 'QuotaExceededError' && + (await this._makeRoom() + .then(() => this._store.set(hash, content)) + .then( + () => true, + () => false, + )); + if (!stored) { + await transact(BLOBS, 'readwrite', (transaction) => + transaction.objectStore(BLOBS).delete(blobKey(this.user, hash)), + ).catch(() => undefined); + return; + } + } this._writtenSinceSweep += content.length; if (!this._sweepTimer && this._writtenSinceSweep > this._options.budgetBytes() / 10) this._sweepTimer = setTimeout(() => void this.sweep().catch(() => undefined), 30_000); } + /** + * One sweep at a time for room, joined by every write that ran out of it meanwhile. Their + * records are already written, so the sweep counts their bytes. + */ + private _makeRoom(): Promise { + this._roomSweep ??= this.sweep() + .catch(() => undefined) + .finally(() => (this._roomSweep = undefined)); + return this._roomSweep; + } + private _remember(hash: string, bytes: Uint8Array): void { if (bytes.length > MEMORY_BYTES / 4) return; const known = this._memory.get(hash); @@ -311,11 +339,25 @@ export class ProjectCaches implements CacheFactory { ).catch(() => undefined); } + /** Keys and values are read in one transaction, so another tab's write can't misalign them. */ private async _trees(): Promise> { const prefix = `${this.user}\0`; - const keys = await keysWithPrefix(TREES, prefix); - const values = await valuesWithPrefix(TREES, prefix); - return new Map(keys.map((key, index) => [key.slice(prefix.length), values[index] ?? []])); + const range = IDBKeyRange.bound(prefix, `${prefix}\uffff`); + let keys: IDBRequest | undefined; + let values: IDBRequest | undefined; + await transact(TREES, 'readonly', (transaction) => { + const store = transaction.objectStore(TREES); + keys = store.getAllKeys(range); + values = store.getAll(range); + return [keys, values]; + }); + const all = values?.result ?? []; + return new Map( + (keys?.result ?? []).map((key, index) => [ + String(key).slice(prefix.length), + all[index] ?? [], + ]), + ); } private async _budget(): Promise { diff --git a/packages/project/src/fs/project-fs.ts b/packages/project/src/fs/project-fs.ts index 75f5a9b4..b992a210 100644 --- a/packages/project/src/fs/project-fs.ts +++ b/packages/project/src/fs/project-fs.ts @@ -31,6 +31,9 @@ export interface WriteOptions { const sameContent = (a: FileEntry, b: FileEntry): boolean => a.hash && b.hash ? a.hash === b.hash : a.mtime === b.mtime && a.size === b.size; +const sameEntry = (a: FileEntry, b: FileEntry): boolean => + a.kind === b.kind && a.size === b.size && a.mtime === b.mtime && a.hash === b.hash; + /** * Files of the open project. The tree is loaded once (cached for fast startup), then kept in * sync by the server change stream. Contents come from the local cache, keyed by the hash the @@ -44,6 +47,17 @@ export class ProjectFs implements Disposable { private readonly _onDidChange = new Emitter(); private readonly _store = new DisposableStore(); private _treeSave: ReturnType | undefined; + /** + * Ticks at each change of the tree. A reply only updates the paths nothing changed since its + * request left: a listing or a read sent before a save must not put the older hash back. + */ + private _clock = 0; + /** Last tick each path changed at. */ + private readonly _changedAt = new Map(); + /** The hash of each path's last save from here: its change coming back is no news. */ + private readonly _saved = new Map(); + /** Last tick each path was removed at, with everything under it. */ + private readonly _removedAt = new Map(); /** Batched changes (from the server watcher and this client's own writes). */ readonly onDidChange: Event = this._onDidChange.event; @@ -80,8 +94,19 @@ export class ProjectFs implements Disposable { /** Re-fetches the whole tree and reports the differences as changes. */ async refresh(): Promise { + const sent = this._clock; const entries = await this._fs.list({ project: this.projectId, path: '', recursive: true }); const next = new Map(entries.map((entry) => [entry.path, entry])); + /** + * What changed since the listing left stays as the tree has it. A path the tree does not + * have yet (a change waiting for its stat) takes the listing's entry, unless it was removed. + */ + for (const path of new Set([...next.keys(), ...this._tree.keys()])) { + if (!this._changedSince(path, sent)) continue; + const current = this._tree.get(path); + if (current) next.set(path, current); + else if (this._removedSince(path, sent)) next.delete(path); + } const changes: FileChange[] = []; for (const [path, entry] of next) { const previous = this._tree.get(path); @@ -93,10 +118,17 @@ export class ProjectFs implements Disposable { for (const [path, entry] of this._tree) { if (!next.has(path)) changes.push({ type: 'deleted', path, kind: entry.kind }); } + const tick = ++this._clock; + for (const [path, entry] of this._tree) { + const listed = next.get(path); + if (!listed) this._removedAt.set(path, tick); + else if (!sameEntry(entry, listed)) this._changedAt.set(path, tick); + } + for (const path of next.keys()) if (!this._tree.has(path)) this._changedAt.set(path, tick); this._tree.clear(); for (const [path, entry] of next) this._tree.set(path, entry); this._ready.set(true); - await this._treeCache.save(entries); + await this._treeCache.save([...next.values()]); if (changes.length) this._onDidChange.fire(changes); } @@ -164,24 +196,37 @@ export class ProjectFs implements Disposable { content: Uint8Array | string, options: WriteOptions = {}, ): Promise { - const result = await this._fs.write({ - project: this.projectId, - path, - content, - ...(options.expectedHash !== undefined && { expectedHash: options.expectedHash }), - }); const bytes = typeof content === 'string' ? new TextEncoder().encode(content) : content; + /** Known before the save leaves: its own change from the watcher may come back first. */ + const hash = await sha1Hex(bytes).catch(() => undefined); + const savedBefore = this._saved.get(path); + if (hash) this._saved.set(path, hash); + const result = await this._fs + .write({ + project: this.projectId, + path, + content, + ...(options.expectedHash !== undefined && { expectedHash: options.expectedHash }), + }) + .catch((error: unknown) => { + if (savedBefore === undefined) this._saved.delete(path); + else this._saved.set(path, savedBefore); + throw error; + }); const existed = this._tree.has(path); - this._tree.set(path, { + this._saved.set(path, result.hash); + this._setEntry(path, { path, kind: 'file', size: result.size, mtime: result.mtime, hash: result.hash, }); - /** Kept only if the server wrote exactly these bytes: the cache must never lie. */ - if ((await sha1Hex(bytes).catch(() => undefined)) === result.hash) - await this._contents.set(result.hash, bytes).catch(() => undefined); + /** + * Kept only if the server wrote exactly these bytes: the cache must never lie. Stored in the + * background: the file is saved, and the cache keeps the bytes in memory at once. + */ + if (hash === result.hash) void this._contents.set(result.hash, bytes).catch(() => undefined); this._scheduleTreeSave(); this._onDidChange.fire([{ type: existed ? 'changed' : 'created', path, kind: 'file' }]); return result; @@ -189,7 +234,7 @@ export class ProjectFs implements Disposable { async mkdir(path: string): Promise { const entry = await this._fs.mkdir({ project: this.projectId, path }); - this._tree.set(path, entry); + this._setEntry(path, entry); } async rename(from: string, to: string, overwrite = false): Promise { @@ -234,20 +279,47 @@ export class ProjectFs implements Disposable { private async _applyRemote(changes: readonly FileChange[]): Promise { const relevant: FileChange[] = []; + /** + * What requests already on their way say of these paths is older than these changes, and + * the hash the tree has is no longer known to be current: until the stat answers, reads go to + * the server. `before` keeps the entries to tell whether the content changed. + */ + const before = new Map(); + for (const change of changes) { + this._changedAt.set(change.path, ++this._clock); + const current = this._tree.get(change.path); + if (!current?.hash) continue; + before.set(change.path, current); + this._tree.set(change.path, { ...current, hash: undefined }); + } + /** + * Every change is checked with a stat, deletions included: events come late and out of order + * (the echo of a deletion after the file was made again), the stat tells what is there now. + */ for (const change of changes) { - if (change.type === 'deleted') { + const sent = this._clock; + const entry = await this._fs + .stat({ project: this.projectId, path: change.path }) + .catch(() => null); + if (!entry) { + if (change.type !== 'deleted' || this._changedSince(change.path, sent)) continue; + const prefix = `${change.path}/`; + if (![...this._tree.keys()].some((key) => key === change.path || key.startsWith(prefix))) + continue; this._removeFromTree(change.path); relevant.push(change); continue; } - const entry = await this._fs - .stat({ project: this.projectId, path: change.path }) - .catch(() => null); - if (!entry) continue; - const previous = this._tree.get(change.path); - this._tree.set(change.path, entry); - if (previous && entry.kind === 'file' && sameContent(previous, entry)) continue; - relevant.push(change); + const previous = before.get(change.path) ?? this._tree.get(change.path); + /** A newer update already has the path: a read's is still told, a save told itself. */ + if (!this._changedSince(change.path, sent)) this._setEntry(change.path, entry); + const now = this._tree.get(change.path); + /** A save's own echo is absorbed once; any other content lands and forgets the save. */ + const saved = this._saved.get(change.path); + this._saved.delete(change.path); + if (!now || (previous && now.kind === 'file' && sameContent(previous, now))) continue; + if (now.hash !== undefined && now.hash === saved) continue; + relevant.push({ type: previous ? 'changed' : 'created', path: change.path, kind: now.kind }); } this._scheduleTreeSave(); if (relevant.length) this._onDidChange.fire(relevant); @@ -290,6 +362,7 @@ export class ProjectFs implements Disposable { const readChunk = async (chunk: typeof ask) => { let pending = chunk; while (pending.length) { + const sent = this._clock; const answer = await this._fs.readMany({ project: this.projectId, files: pending }); for (const result of answer.files) { if ('error' in result) { @@ -303,7 +376,7 @@ export class ProjectFs implements Disposable { mtime: result.mtime, hash: result.hash, }; - this._tree.set(result.path, entry); + if (!this._changedSince(result.path, sent)) this._setEntry(result.path, entry); if ('notModified' in result) { const bytes = cached.get(result.path); if (bytes) found.set(result.path, { ...entry, content: bytes, hash: result.hash }); @@ -344,7 +417,26 @@ export class ProjectFs implements Disposable { }, 1000); } + private _setEntry(path: string, entry: FileEntry): void { + this._tree.set(path, entry); + this._changedAt.set(path, ++this._clock); + } + + /** Whether the tree changed at `path` (or removed a folder holding it) after tick `sent`. */ + private _changedSince(path: string, sent: number): boolean { + return (this._changedAt.get(path) ?? 0) > sent || this._removedSince(path, sent); + } + + /** Whether `path`, or a folder holding it, left the tree after tick `sent`. */ + private _removedSince(path: string, sent: number): boolean { + for (let at = path; ; at = at.slice(0, at.lastIndexOf('/'))) { + if ((this._removedAt.get(at) ?? 0) > sent) return true; + if (!at.includes('/')) return false; + } + } + private _removeFromTree(path: string): void { + this._removedAt.set(path, ++this._clock); for (const key of [...this._tree.keys()]) { if (key === path || key.startsWith(`${path}/`)) this._tree.delete(key); } diff --git a/packages/project/test/cache/project-caches.browser.spec.ts b/packages/project/test/cache/project-caches.browser.spec.ts index d1b76de5..1dcacfc8 100644 --- a/packages/project/test/cache/project-caches.browser.spec.ts +++ b/packages/project/test/cache/project-caches.browser.spec.ts @@ -130,4 +130,39 @@ describe('ProjectCaches', () => { expect(user).toBeTruthy(); await caches.clearAll(); }); + + it('runs one sweep for every write the full quota refused at once', async () => { + const { caches } = setup(); + const store = (caches as unknown as { _store: OpfsContentCache })._store; + const set = store.set.bind(store); + const refused = new Set(); + store.set = async (hash, bytes) => { + if (!refused.has(hash)) { + refused.add(hash); + throw Object.assign(new Error('full'), { name: 'QuotaExceededError' }); + } + return set(hash, bytes); + }; + const sweep = caches.sweep.bind(caches); + let sweeps = 0; + caches.sweep = (options) => { + sweeps++; + return sweep(options); + }; + const items = await Promise.all(['a', 'b', 'c'].map(content)); + await Promise.all(items.map((item) => caches.contents().set(item.hash, item.bytes))); + expect(sweeps).toBe(1); + for (const item of items) expect(await store.get(item.hash)).toEqual(item.bytes); + await caches.clearAll(); + }); + + it('counts no content the browser could not keep', async () => { + const { caches } = setup(); + const store = (caches as unknown as { _store: OpfsContentCache })._store; + store.set = () => Promise.reject(new Error('broken')); + const item = await content('lost'); + await caches.contents().set(item.hash, item.bytes); + expect((await caches.usage()).bytes).toBe(0); + await caches.clearAll(); + }); }); diff --git a/packages/project/test/fs/project-fs-races.spec.ts b/packages/project/test/fs/project-fs-races.spec.ts new file mode 100644 index 00000000..cdc68fa7 --- /dev/null +++ b/packages/project/test/fs/project-fs-races.spec.ts @@ -0,0 +1,371 @@ +import { createHash } from 'node:crypto'; +import { describe, expect, it } from 'vitest'; + +import type { FileChange, FileEntry } from '@nanoforge-dev/editor-protocol'; +import type { RpcClient } from '@nanoforge-dev/editor-rpc'; + +import type { ContentCache } from '../../src/cache/cache.type'; +import { MemoryContentCache } from '../../src/cache/memory.content-cache'; +import { MemoryTreeCache } from '../../src/cache/memory.tree-cache'; +import { ProjectFs } from '../../src/fs/project-fs'; + +const encoder = new TextEncoder(); +const sha1 = (text: string) => createHash('sha1').update(text).digest('hex'); +const fileEntry = (path: string, text: string): FileEntry & { hash: string } => ({ + path, + kind: 'file', + size: text.length, + mtime: 1, + hash: sha1(text), +}); + +/** A reply the test sends when it chooses. */ +const held = () => { + let release!: (value: T) => void; + const promise = new Promise((resolve) => (release = resolve)); + return { promise, release }; +}; + +/** + * A server whose answers the test can hold back: `list`, `readMany` and `stat` take the next + * queued answer when there is one, else answer from `files`; `write` waits for the next queued + * release, when there is one. + */ +const fakeServer = (initial: Record) => { + const files = new Map(Object.entries(initial)); + const lists: Promise[] = []; + const readManys: Promise[] = []; + const stats: Promise[] = []; + const writes: Promise[] = []; + let onChanges: ((event: { changes: FileChange[] }) => void) | undefined; + const fs = { + list: () => + lists.shift() ?? + Promise.resolve([...files].map(([path, text]) => fileEntry(path, text) as FileEntry)), + stat: ({ path }: { path: string }) => + stats.shift() ?? Promise.resolve(files.has(path) ? fileEntry(path, files.get(path)!) : null), + readMany: ({ files: wanted }: { files: { path: string }[] }) => + readManys.shift() ?? + Promise.resolve({ + files: wanted.map(({ path }) => ({ + ...fileEntry(path, files.get(path)!), + content: encoder.encode(files.get(path)!), + })), + rest: [], + }), + delete: ({ path }: { path: string }) => { + files.delete(path); + return Promise.resolve({ trashPath: `.trash/${path}` }); + }, + write: ({ path, content }: { path: string; content: string }) => { + files.set(path, content); + return (writes.shift() ?? Promise.resolve()).then(() => fileEntry(path, content)); + }, + }; + const rpc = { + api: () => fs, + subscribe: ( + _contract: unknown, + _name: string, + _input: unknown, + listener: (event: { changes: FileChange[] }) => void, + ) => { + onChanges = listener; + return { dispose: () => undefined }; + }, + onDidReconnect: () => ({ dispose: () => undefined }), + } as unknown as RpcClient; + return { + rpc, + files, + lists, + readManys, + stats, + writes, + change: (changes: FileChange[]) => onChanges?.({ changes }), + }; +}; + +const settle = () => new Promise((resolve) => setTimeout(resolve, 20)); + +const open = async (server: ReturnType, contents: ContentCache) => { + const fs = new ProjectFs('p', server.rpc, contents, new MemoryTreeCache()); + await fs.initialize(); + return fs; +}; + +describe('ProjectFs races', () => { + it('keeps a save made while a listing was on its way', async () => { + const server = fakeServer({ 'a.ts': 'old' }); + const contents = new MemoryContentCache(); + await contents.set(sha1('old'), encoder.encode('old')); + const fs = await open(server, contents); + const listing = held(); + server.lists.push(listing.promise); + const refreshed = fs.refresh(); + await fs.write('a.ts', 'new'); + listing.release([fileEntry('a.ts', 'old')]); + await refreshed; + expect(fs.entry('a.ts')?.hash).toBe(sha1('new')); + expect((await fs.readText('a.ts')).text).toBe('new'); + fs.dispose(); + }); + + it('keeps a change from the server made while a listing was on its way', async () => { + const server = fakeServer({ 'a.ts': 'old' }); + const contents = new MemoryContentCache(); + await contents.set(sha1('old'), encoder.encode('old')); + await contents.set(sha1('new'), encoder.encode('new')); + const fs = await open(server, contents); + const listing = held(); + server.lists.push(listing.promise); + const refreshed = fs.refresh(); + server.files.set('a.ts', 'new'); + server.change([{ type: 'changed', path: 'a.ts', kind: 'file' }]); + await settle(); + listing.release([fileEntry('a.ts', 'old')]); + await refreshed; + expect(fs.entry('a.ts')?.hash).toBe(sha1('new')); + expect((await fs.readText('a.ts')).text).toBe('new'); + fs.dispose(); + }); + + it('keeps a change from the server whose stat comes back after a listing', async () => { + const server = fakeServer({ 'a.ts': 'old' }); + const contents = new MemoryContentCache(); + await contents.set(sha1('new'), encoder.encode('new')); + const fs = await open(server, contents); + const listing = held(); + const stat = held(); + server.lists.push(listing.promise); + server.stats.push(stat.promise); + const refreshed = fs.refresh(); + server.change([{ type: 'changed', path: 'a.ts', kind: 'file' }]); + await settle(); + listing.release([fileEntry('a.ts', 'old')]); + await refreshed; + stat.release(fileEntry('a.ts', 'new')); + await settle(); + expect(fs.entry('a.ts')?.hash).toBe(sha1('new')); + fs.dispose(); + }); + + it('keeps a new file a listing shows while the stat of its creation is on its way', async () => { + const server = fakeServer({ 'a.ts': 'a' }); + const fs = await open(server, new MemoryContentCache()); + const listing = held(); + const stat = held(); + server.lists.push(listing.promise); + server.stats.push(stat.promise); + const refreshed = fs.refresh(); + server.files.set('b.ts', 'b'); + server.change([{ type: 'created', path: 'b.ts', kind: 'file' }]); + await settle(); + listing.release([fileEntry('a.ts', 'a'), fileEntry('b.ts', 'b')]); + await refreshed; + expect(fs.entry('b.ts')).toBeDefined(); + stat.release(fileEntry('b.ts', 'b')); + await settle(); + expect(fs.entry('b.ts')?.hash).toBe(sha1('b')); + fs.dispose(); + }); + + it('reads a file the server changed from the server until its stat comes back', async () => { + const server = fakeServer({ 'a.ts': 'old' }); + const contents = new MemoryContentCache(); + await contents.set(sha1('old'), encoder.encode('old')); + const fs = await open(server, contents); + const changes: FileChange[] = []; + const subscription = fs.onDidChange((batch) => changes.push(...batch)); + const listing = held(); + const stat = held(); + server.lists.push(listing.promise); + server.stats.push(stat.promise); + /** Same size and date: only the hash tells the copies apart. */ + server.files.set('a.ts', 'new'); + const refreshed = fs.refresh(); + server.change([{ type: 'changed', path: 'a.ts', kind: 'file' }]); + await settle(); + listing.release([fileEntry('a.ts', 'new')]); + await refreshed; + expect((await fs.readText('a.ts')).text).toBe('new'); + stat.release(fileEntry('a.ts', 'new')); + await settle(); + expect(fs.entry('a.ts')?.hash).toBe(sha1('new')); + expect(changes).toContainEqual({ type: 'changed', path: 'a.ts', kind: 'file' }); + subscription.dispose(); + fs.dispose(); + }); + + it('tells a save once, when its own change comes back before the save answers', async () => { + const server = fakeServer({ 'a.ts': 'old' }); + const fs = await open(server, new MemoryContentCache()); + const changes: FileChange[] = []; + const subscription = fs.onDidChange((batch) => changes.push(...batch)); + const stat = held(); + server.stats.push(stat.promise); + const saved = fs.write('a.ts', 'new'); + server.change([{ type: 'changed', path: 'a.ts', kind: 'file' }]); + await saved; + stat.release(fileEntry('a.ts', 'new')); + await settle(); + expect(changes).toEqual([{ type: 'changed', path: 'a.ts', kind: 'file' }]); + expect(fs.entry('a.ts')?.hash).toBe(sha1('new')); + subscription.dispose(); + fs.dispose(); + }); + + it('tells a save once, when the stat of its own change answers before the save', async () => { + const server = fakeServer({ 'a.ts': 'old' }); + const fs = await open(server, new MemoryContentCache()); + const changes: FileChange[] = []; + const subscription = fs.onDidChange((batch) => changes.push(...batch)); + const answer = held(); + server.writes.push(answer.promise); + const saved = fs.write('a.ts', 'new'); + await settle(); + server.change([{ type: 'changed', path: 'a.ts', kind: 'file' }]); + await settle(); + answer.release(undefined); + await saved; + await settle(); + expect(changes).toEqual([{ type: 'changed', path: 'a.ts', kind: 'file' }]); + expect(fs.entry('a.ts')?.hash).toBe(sha1('new')); + subscription.dispose(); + fs.dispose(); + }); + + it('keeps a file made again when the event of its earlier deletion comes late', async () => { + const server = fakeServer({ 'a.ts': 'a', 'b.ts': 'b' }); + const fs = await open(server, new MemoryContentCache()); + await fs.delete('b.ts'); + server.files.set('b.ts', 'b'); + const listing = held(); + server.lists.push(listing.promise); + const refreshed = fs.refresh(); + server.change([{ type: 'deleted', path: 'b.ts', kind: 'file' }]); + await settle(); + listing.release([fileEntry('a.ts', 'a'), fileEntry('b.ts', 'b')]); + await refreshed; + expect(fs.entry('b.ts')).toBeDefined(); + fs.dispose(); + }); + + it('keeps a file made again when the event of its earlier deletion comes after a listing', async () => { + const server = fakeServer({ 'a.ts': 'a', 'b.ts': 'b' }); + const fs = await open(server, new MemoryContentCache()); + await fs.delete('b.ts'); + server.files.set('b.ts', 'b'); + await fs.refresh(); + server.change([{ type: 'deleted', path: 'b.ts', kind: 'file' }]); + await settle(); + expect(fs.entry('b.ts')).toBeDefined(); + fs.dispose(); + }); + + it('does not bring back a file made and deleted at once', async () => { + const server = fakeServer({ 'a.ts': 'a' }); + const fs = await open(server, new MemoryContentCache()); + const stat = held(); + server.stats.push(stat.promise); + server.change([{ type: 'created', path: 'tmp.ts', kind: 'file' }]); + await settle(); + server.change([{ type: 'deleted', path: 'tmp.ts', kind: 'file' }]); + await settle(); + stat.release(fileEntry('tmp.ts', 'x')); + await settle(); + expect(fs.entry('tmp.ts')).toBeUndefined(); + fs.dispose(); + }); + + it('tells an outside change back to what the editor saved', async () => { + const server = fakeServer({ 'a.ts': 'old' }); + const fs = await open(server, new MemoryContentCache()); + await fs.write('a.ts', 'mine'); + await settle(); + const changes: FileChange[] = []; + const subscription = fs.onDidChange((batch) => changes.push(...batch)); + for (const text of ['theirs', 'mine']) { + server.files.set('a.ts', text); + server.change([{ type: 'changed', path: 'a.ts', kind: 'file' }]); + await settle(); + } + expect(changes).toEqual([ + { type: 'changed', path: 'a.ts', kind: 'file' }, + { type: 'changed', path: 'a.ts', kind: 'file' }, + ]); + subscription.dispose(); + fs.dispose(); + }); + + it('keeps a save made while a read was on its way', async () => { + const server = fakeServer({ 'a.ts': 'old' }); + const fs = await open(server, new MemoryContentCache()); + const answer = held(); + server.readManys.push(answer.promise); + const read = fs.readText('a.ts'); + /** The read leaves (after its cache lookups) before the save. */ + await settle(); + await fs.write('a.ts', 'new'); + answer.release({ + files: [{ ...fileEntry('a.ts', 'old'), content: encoder.encode('old') }], + rest: [], + }); + expect((await read).text).toBe('old'); + expect(fs.entry('a.ts')?.hash).toBe(sha1('new')); + expect((await fs.readText('a.ts')).text).toBe('new'); + fs.dispose(); + }); + + it('keeps the newer of two reads that overlap', async () => { + const server = fakeServer({ 'a.ts': 'old' }); + const fs = await open(server, new MemoryContentCache()); + const first = held(); + server.readManys.push(first.promise); + const older = fs.readText('a.ts'); + await settle(); + server.files.set('a.ts', 'new'); + expect((await fs.readText('a.ts', { fresh: true })).text).toBe('new'); + first.release({ + files: [{ ...fileEntry('a.ts', 'old'), content: encoder.encode('old') }], + rest: [], + }); + await older; + expect(fs.entry('a.ts')?.hash).toBe(sha1('new')); + expect((await fs.readText('a.ts')).text).toBe('new'); + fs.dispose(); + }); + + it('does not bring back a folder deleted while a listing was on its way', async () => { + const server = fakeServer({ 'd/a.ts': 'a' }); + const fs = await open(server, new MemoryContentCache()); + const listing = held(); + server.lists.push(listing.promise); + const refreshed = fs.refresh(); + server.change([{ type: 'deleted', path: 'd', kind: 'directory' }]); + await settle(); + listing.release([ + { path: 'd', kind: 'directory', size: 0, mtime: 1 }, + fileEntry('d/a.ts', 'a'), + fileEntry('d/b.ts', 'b'), + ]); + await refreshed; + expect(fs.entries.map((entry) => entry.path)).toEqual([]); + fs.dispose(); + }); + + it('saves without waiting for the content cache', async () => { + const server = fakeServer({ 'a.ts': 'old' }); + const stuck: ContentCache = { + get: () => Promise.resolve(undefined), + set: () => new Promise(() => undefined), + }; + const fs = await open(server, stuck); + const saved = await Promise.race([ + fs.write('a.ts', 'new').then(() => 'saved'), + new Promise((resolve) => setTimeout(() => resolve('waiting'), 200)), + ]); + expect(saved).toBe('saved'); + fs.dispose(); + }); +}); diff --git a/packages/project/test/fs/project-fs.spec.ts b/packages/project/test/fs/project-fs.spec.ts index 027c50c6..4b32e8aa 100644 --- a/packages/project/test/fs/project-fs.spec.ts +++ b/packages/project/test/fs/project-fs.spec.ts @@ -144,7 +144,8 @@ describe('ProjectFs', () => { utimesSync(file, before.atime, before.mtime); await project.fs.refresh(); expect((await project.fs.readText(path)).text).not.toBe(text); - expect(reads).toEqual([path]); + /** Read again, maybe twice: the watcher's change may still wait for its stat. */ + expect([...new Set(reads)]).toEqual([path]); reads.length = 0; await project.fs.rename(path, 'apps/client/src/renamed.ts'); diff --git a/packages/server-core/src/fs/file-hashes.ts b/packages/server-core/src/fs/file-hashes.ts index a1a0ca0d..183d6500 100644 --- a/packages/server-core/src/fs/file-hashes.ts +++ b/packages/server-core/src/fs/file-hashes.ts @@ -3,7 +3,7 @@ import { createReadStream } from 'node:fs'; import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; import { dirname } from 'node:path'; -/** Hashes kept at most; the oldest are forgotten first. */ +/** Hashes kept at most; the least recently used are forgotten first. */ const MAX_ENTRIES = 50_000; /** Saving waits this long after the last new hash. */ const SAVE_DELAY_MS = 2000; @@ -28,12 +28,16 @@ export interface FileHashesOptions { } /** - * sha1 of files, kept by path and version (size, mtime, ctime, inode): a listing hashes each - * file once, and again only after it changed. Files are read as streams, so memory stays small - * whatever their size. Clients compare the hash with the copy they cached. + * sha1 of files, one per path, valid for one version (size, mtime, ctime, inode): a listing + * hashes each file once, and again only after it changed. Files are read as streams, so memory + * stays small whatever their size. Clients compare the hash with the copy they cached. + * + * A file written between its stat and the end of its read gets the new content's hash under the + * old version: the next write changes the version again, and reads hash what they send anyway. */ export class FileHashes { - private readonly _hashes = new Map(); + /** By absolute path, least recently used first. */ + private readonly _hashes = new Map(); private _loaded: Promise | undefined; private _saveTimer: ReturnType | undefined; @@ -43,9 +47,12 @@ export class FileHashes { async of(absolute: string, info: FileVersion): Promise { if (info.size > this._options.maxBytes) return undefined; await this._load(); - const key = keyOf(absolute, info); - const known = this._hashes.get(key); - if (known) return known; + const known = this._hashes.get(absolute); + if (known?.version === versionOf(info)) { + this._hashes.delete(absolute); + this._hashes.set(absolute, known); + return known.hash; + } const hash = await streamSha1(absolute); this.remember(absolute, info, hash); return hash; @@ -53,9 +60,11 @@ export class FileHashes { /** Records a hash computed elsewhere (a read or a write of the file). */ remember(absolute: string, info: FileVersion, hash: string): void { - const key = keyOf(absolute, info); - this._hashes.delete(key); - this._hashes.set(key, hash); + const version = versionOf(info); + const known = this._hashes.get(absolute); + this._hashes.delete(absolute); + this._hashes.set(absolute, { version, hash }); + if (known?.version === version && known.hash === hash) return; if (this._hashes.size > MAX_ENTRIES) this._hashes.delete(this._hashes.keys().next().value as string); this._scheduleSave(); @@ -74,8 +83,10 @@ export class FileHashes { this._loaded ??= file ? readFile(file, 'utf8') .then((text) => { - for (const [key, hash] of JSON.parse(text) as [string, string][]) - if (!this._hashes.has(key)) this._hashes.set(key, hash); + for (const [path, value] of JSON.parse(text) as [string, unknown][]) { + if (this._hashes.has(path) || !isSaved(value)) continue; + this._hashes.set(path, { version: value.version, hash: value.hash }); + } }) .catch(() => undefined) : Promise.resolve(); @@ -103,8 +114,15 @@ export class FileHashes { } } -const keyOf = (absolute: string, info: FileVersion) => - `${absolute}\0${info.size}\0${info.mtimeMs}\0${info.ctimeMs}\0${info.ino}`; +const versionOf = (info: FileVersion) => + `${info.size}\0${info.mtimeMs}\0${info.ctimeMs}\0${info.ino}`; + +/** An entry of the saved file (older files, keyed by version, are dropped). */ +const isSaved = (value: unknown): value is { version: string; hash: string } => + typeof value === 'object' && + value !== null && + typeof (value as { version?: unknown }).version === 'string' && + typeof (value as { hash?: unknown }).hash === 'string'; const streamSha1 = (absolute: string): Promise => new Promise((resolve, reject) => { diff --git a/packages/server-core/src/fs/fs-service.ts b/packages/server-core/src/fs/fs-service.ts index f3827fac..d8ac5df9 100644 --- a/packages/server-core/src/fs/fs-service.ts +++ b/packages/server-core/src/fs/fs-service.ts @@ -120,7 +120,10 @@ export class ProjectFileSystem { try { const info = await stat(absolute); const kind = info.isDirectory() ? 'directory' : 'file'; - const hash = kind === 'file' ? await this._hashes.of(absolute, info) : undefined; + /** A file that can't be read (permissions, a lock) or a pipe gets no hash, not an error. */ + const hash = info.isFile() + ? await this._hashes.of(absolute, info).catch(() => undefined) + : undefined; return { path, kind, size: info.size, mtime: info.mtimeMs, ...(hash && { hash }) }; } catch (error) { if (errno(error) === 'ENOENT') return null; @@ -149,7 +152,8 @@ export class ProjectFileSystem { if (!info) continue; const kind = info.isDirectory() ? 'directory' : 'file'; if (this.isIgnored(childRel, kind === 'directory')) continue; - if (kind === 'file') + /** Pipes would block the read; the trash is never read, so its files cost nothing. */ + if (info.isFile() && !inTrash(childRel)) files.push({ index: entries.length, absolute: join(dir, entry.name), info }); entries.push({ path: childRel, kind, size: info.size, mtime: info.mtimeMs }); if (recursive && kind === 'directory' && !SKIPPED_IN_RECURSIVE_LIST.has(entry.name)) { @@ -299,7 +303,7 @@ export class ProjectFileSystem { * place in the trash, for `restore`. */ async delete(path: string): Promise<{ trashPath: string }> { - if (path === '' || path === TRASH_DIR || path.startsWith(`${TRASH_DIR}/`)) { + if (path === '' || inTrash(path)) { throw new RpcError('BAD_REQUEST', `Cannot delete ${path || 'the project root'}`); } assertWritable(path); @@ -360,3 +364,5 @@ export class ProjectFileSystem { /** Heavy folders a recursive listing does not descend into. */ const SKIPPED_IN_RECURSIVE_LIST = new Set(['node_modules']); + +const inTrash = (path: string) => path === TRASH_DIR || path.startsWith(`${TRASH_DIR}/`); diff --git a/packages/server-core/test/fs/file-hashes.spec.ts b/packages/server-core/test/fs/file-hashes.spec.ts index 4bb7955b..fbeb22bf 100644 --- a/packages/server-core/test/fs/file-hashes.spec.ts +++ b/packages/server-core/test/fs/file-hashes.spec.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, describe, expect, it } from 'vitest'; @@ -29,4 +29,36 @@ describe('FileHashes', () => { writeFileSync(file, 'x'.repeat(2048)); expect(await new FileHashes({ maxBytes: 1024 }).of(file, statSync(file))).toBeUndefined(); }); + + it('keeps one hash per file, and saves nothing when a hash did not change', async () => { + const file = join(dir, 'edited.ts'); + const store = join(dir, 'one-per-file/file-hashes.json'); + const hashes = new FileHashes({ maxBytes: 1024, file: store }); + writeFileSync(file, 'one'); + await hashes.of(file, statSync(file)); + writeFileSync(file, 'two, longer'); + const info = statSync(file); + expect(await hashes.of(file, info)).toBe(sha1('two, longer')); + await hashes.flush(); + expect(JSON.parse(readFileSync(store, 'utf8'))).toHaveLength(1); + + rmSync(store); + hashes.remember(file, info, sha1('two, longer')); + await hashes.of(file, info); + await hashes.flush(); + expect(existsSync(store)).toBe(false); + }); + + it('forgets the least recently used hash first', async () => { + const file = join(dir, 'used.ts'); + writeFileSync(file, 'used'); + const info = statSync(file); + const hashes = new FileHashes({ maxBytes: 1024 }); + hashes.remember(file, info, 'remembered'); + for (let index = 1; index < 50_000; index++) + hashes.remember(join(dir, `other-${index}`), info, 'other'); + expect(await hashes.of(file, info)).toBe('remembered'); + hashes.remember(join(dir, 'one-more'), info, 'other'); + expect(await hashes.of(file, info)).toBe('remembered'); + }); }); diff --git a/packages/server-core/test/fs/fs-service.spec.ts b/packages/server-core/test/fs/fs-service.spec.ts index 087320de..65dee954 100644 --- a/packages/server-core/test/fs/fs-service.spec.ts +++ b/packages/server-core/test/fs/fs-service.spec.ts @@ -1,4 +1,6 @@ +import { execFileSync } from 'node:child_process'; import { + chmodSync, existsSync, mkdirSync, mkdtempSync, @@ -45,6 +47,47 @@ describe('ProjectFileSystem', () => { expect((await fs.stat('src/main.ts'))?.hash).toBe(sha1('export [];\n')); }); + it.skipIf(process.platform === 'win32' || process.getuid?.() === 0)( + 'gives no hash to a file it cannot read, and still stats it after a rename', + async () => { + const { root, fs } = project(); + writeFileSync(join(root, 'src/locked.ts'), 'x'); + chmodSync(join(root, 'src/locked.ts'), 0o000); + try { + expect(await fs.stat('src/locked.ts')).toMatchObject({ + path: 'src/locked.ts', + kind: 'file', + }); + expect((await fs.stat('src/locked.ts'))?.hash).toBeUndefined(); + expect(await fs.rename('src/locked.ts', 'src/moved.ts')).toMatchObject({ + path: 'src/moved.ts', + }); + const listed = await fs.list('', true); + expect(listed.find((entry) => entry.path === 'src/moved.ts')?.hash).toBeUndefined(); + } finally { + for (const name of ['locked.ts', 'moved.ts']) { + if (existsSync(join(root, 'src', name))) chmodSync(join(root, 'src', name), 0o644); + } + } + }, + ); + + it.skipIf(process.platform === 'win32')('lists a named pipe without reading it', async () => { + const { root, fs } = project(); + execFileSync('mkfifo', [join(root, 'src/pipe')]); + const listed = await fs.list('', true); + expect(listed.find((entry) => entry.path === 'src/pipe')?.hash).toBeUndefined(); + expect((await fs.stat('src/pipe'))?.hash).toBeUndefined(); + }); + + it('gives no hash to the files in the trash', async () => { + const { fs } = project(); + const { trashPath } = await fs.delete('src/main.ts'); + const listed = await fs.list('', true); + expect(listed.find((entry) => entry.path === trashPath)).toBeDefined(); + expect(listed.find((entry) => entry.path === trashPath)?.hash).toBeUndefined(); + }); + it('answers readMany within 8 MiB, the rest for the next call', async () => { const { root, fs } = project(); const big = new Uint8Array(5 * 1024 * 1024); diff --git a/plugins/code-editor/test/typing/type-libraries.spec.ts b/plugins/code-editor/test/typing/type-libraries.spec.ts index 0d4eceee..8325ea6e 100644 --- a/plugins/code-editor/test/typing/type-libraries.spec.ts +++ b/plugins/code-editor/test/typing/type-libraries.spec.ts @@ -18,6 +18,7 @@ const setup = (count: number) => { fs: { entries: paths.map((path) => ({ kind: 'file', path })), onDidChange: () => ({ dispose: () => undefined }), + readMany: async () => undefined, readText: (path: string) => { const read = deferred<{ text: string }>(); reads.set(path, read); @@ -47,6 +48,7 @@ const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); describe('TypeLibraries', () => { it("adds none of the old project's files once disposed", async () => { const { reads, added, libraries } = setup(20); + await settle(); expect(reads.size).toBe(8); libraries.dispose(); for (const read of reads.values()) read.resolve({ text: 'export const a = 1;\n' }); @@ -57,6 +59,7 @@ describe('TypeLibraries', () => { it('adds no package declarations that arrive after dispose', async () => { const { reads, added, libraries, types } = setup(1); + await settle(); reads.get('apps/client/src/file-0.ts')!.resolve({ text: "import { a } from 'speed';\n" }); await settle(); expect(added).toEqual(['file:///apps/client/src/file-0.ts']);