feat: add cache system with OPFS

.github/workflows/e2e.yml

@@ -54,7 +54,7 @@ jobs:
         working-directory: engine
 
       - name: Install browsers
-        run: pnpm --filter @nanoforge-dev/editor exec playwright install --with-deps chromium
+        run: pnpm --filter @nanoforge-dev/editor exec playwright install --with-deps chromium firefox
 
       - name: Run end-to-end tests
         run: pnpm --filter @nanoforge-dev/editor test:e2e

.github/workflows/tests.yml

@@ -34,8 +34,10 @@ jobs:
       - name: Run type checks
         run: pnpm typecheck
 
-      - name: Install browsers for component tests
-        run: pnpm --filter @nanoforge-dev/editor-ui exec playwright install --with-deps chromium
+      - name: Install browsers for component and cache tests
+        run: pnpm --filter @nanoforge-dev/editor-ui exec playwright install --with-deps chromium firefox
 
       - name: Run tests
         run: pnpm test:unit
+        env:
+          BROWSERS: chromium,firefox

CLAUDE.md

@@ -259,6 +259,7 @@ Everything in our registry has a `nanoforge.manifest.json` and a name `@scope/na
   - The server worker exits on `disconnect`.
 - **CLI:** `nf` delegates to the CLI installed in the project (`<cwd>/node_modules/@nanoforge-dev/cli/dist`), so `NF_CLI_PATH` does not change which build code runs for such a project.
 - **Console filters and `e2e`:** the Console's group, source and level filters are widget state (saved server-side). A test that changes them puts them back and waits 900 ms.
+- **Browser file cache (M8):** contents are kept by sha1 (`FileEntry.hash`, computed by the server's `FileHashes`, keyed by path, size, mtime, ctime and inode) in OPFS `nanoforge-editor/v2/<user>/blobs/`, and the cached tree is the path → hash index (`ProjectCaches`, `packages/project/src/cache`). Every OPFS read checks the bytes against their hash; a write is cached only when its hash is the server's. All IndexedDB stores of the cache open through `cache-db.ts` (one version). Several reads go through `fs.readMany` (`notModified` for current copies). Browser tests: `packages/project` `*.browser.spec.ts` (Chromium, Firefox; `BROWSERS=…,webkit`), e2e `cache.spec.ts` on Chromium and Firefox. RPC calls are HTTP POSTs (`/rpc/<contract>.<method>`): watch `request`, not WebSocket frames.
 - **File manager:**
   - Fast repeated commands race: they run through a serialized queue.
   - After a drag and drop, focus goes back to the tree (`refocus`), so Ctrl+Z reaches the Files history rather than the layout.

apps/editor/e2e/cache.spec.ts

@@ -0,0 +1,78 @@
+import { expandFolders, openProject, resetLayout } from './helpers';
+import { type Page, expect, test } from './test';
+
+/** Files kept in the browser's cache (OPFS), every user together. */
+const keptFiles = (page: Page) =>
+  page.evaluate(async () => {
+    let count = 0;
+    try {
+      const layout = await (
+        await navigator.storage.getDirectory()
+      )
+        .getDirectoryHandle('nanoforge-editor')
+        .then((root) => root.getDirectoryHandle('v2'));
+      for await (const user of (
+        layout as unknown as { values(): AsyncIterable<FileSystemDirectoryHandle> }
+      ).values()) {
+        const blobs = await user.getDirectoryHandle('blobs');
+        for await (const name of (blobs as unknown as { keys(): AsyncIterable<string> }).keys())
+          if (name) count++;
+      }
+    } catch {}
+    return count;
+  });
+
+const openMain = async (page: Page) => {
+  await expandFolders(page, 'apps', 'client', 'src');
+  await page
+    .getByRole('tree', { name: 'Project files' })
+    .getByRole('treeitem', { name: 'main.ts', exact: true })
+    .dblclick();
+  await expect(page.getByRole('tab', { name: 'main.ts' })).toBeVisible();
+};
+
+test('keeps project files in the browser and reads none of them again after a reload', async ({
+  page,
+}) => {
+  test.setTimeout(90_000);
+  /** File reads the page asks the server for (RPC calls are HTTP requests). */
+  const reads: string[] = [];
+  page.on('request', (request) => {
+    if (request.url().includes('fs.readMany')) reads.push(request.url());
+  });
+  await openProject(page, 'pong');
+  await page.waitForURL(/\/project\//);
+  await resetLayout(page);
+  await openMain(page);
+  await expect.poll(() => keptFiles(page), { timeout: 15_000 }).toBeGreaterThan(5);
+  /** The code worker and the code editor's type information fill the cache in the background. */
+  await page.waitForTimeout(3000);
+  /** The first visit reads the files from the server: the check below can fail. */
+  expect(reads.length).toBeGreaterThan(0);
+
+  reads.length = 0;
+  await page.reload();
+  await expect(page.getByRole('tab', { name: 'main.ts' })).toBeVisible({ timeout: 15_000 });
+  await page.getByRole('tab', { name: 'main.ts' }).click();
+  await expect(page.locator('.monaco-editor .view-lines')).toContainText('NanoforgeFactory', {
+    timeout: 15_000,
+  });
+  await page.waitForTimeout(3000);
+  expect(reads).toEqual([]);
+
+  await page.locator('[data-nf-part="screen"]').click();
+  await page.keyboard.press('Control+Alt+s');
+  const dialog = page.getByRole('dialog', { name: 'Settings' });
+  await dialog
+    .getByRole('tree', { name: 'Settings categories' })
+    .getByRole('treeitem', { name: 'Storage', exact: true })
+    .click();
+  const kept = dialog.getByRole('list', { name: 'Kept projects' });
+  await expect(kept.getByRole('listitem').filter({ hasText: 'pong-network' })).toBeVisible();
+  await dialog.getByRole('button', { name: 'Clear the files kept of pong-network' }).click();
+  await expect(kept.getByRole('listitem').filter({ hasText: 'pong-network' })).toHaveCount(0);
+  await page.keyboard.press('Escape');
+  await page.getByRole('tab', { name: 'main.ts' }).click({ button: 'middle' });
+  await resetLayout(page);
+  await page.waitForTimeout(900);
+});

apps/editor/e2e/project.spec.ts

@@ -95,6 +95,7 @@ test('adds a shared library, chooses the apps that use it, renames and removes i
     .getByRole('checkbox', { name: 'pong-network-client' })
     .uncheck();
   await expect.poll(() => dependencies(CLIENT)).toEqual({});
+  /** The config is written after the package.json. */
   await expect.poll(() => read(CLIENT_CONFIG)).toContain('libs: [],');
   expect(dependencies(SERVER)).toEqual({ '@pong-network/shared': 'workspace:*' });
   await expect(components.getByRole('button', { name: 'C Score', exact: true })).toHaveCount(0);

apps/editor/playwright.config.ts

@@ -13,7 +13,17 @@ if (!/^\.workspace(-[\w-]+)?$/.test(WORKSPACE))
  * fresh workspace holding a copy of pong-network, with the example plugin as a dev plugin.
  * Set CHROME to use an installed Chromium instead of Playwright's, NANOFORGE_ENGINE to play the
  * pong-network example of another engine checkout (default: ../engine next to the editor).
+ * Everything runs on Chromium; the browser cache spec also on Firefox, and on WebKit with
+ * `BROWSERS=…,webkit` where it starts (it needs system libraries some distributions lack).
  */
+const CROSS_BROWSER = 'cache.spec.ts';
+const others = (process.env.BROWSERS ?? 'chromium,firefox')
+  .split(',')
+  .map((browser) => browser.trim())
+  .filter(
+    (browser): browser is 'firefox' | 'webkit' => browser === 'firefox' || browser === 'webkit',
+  );
+
 export default defineConfig({
   testDir: 'e2e',
   testMatch: process.env.BENCH ? '*.bench.ts' : '*.spec.ts',
@@ -24,9 +34,24 @@ export default defineConfig({
   use: {
     baseURL: `http://127.0.0.1:${PORT}`,
     viewport: { width: 1440, height: 860 },
-    launchOptions: process.env.CHROME ? { executablePath: process.env.CHROME } : {},
     trace: 'retain-on-failure',
   },
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        browserName: 'chromium',
+        launchOptions: process.env.CHROME ? { executablePath: process.env.CHROME } : {},
+      },
+    },
+    ...(process.env.BENCH
+      ? []
+      : others.map((browser) => ({
+          name: browser,
+          testMatch: CROSS_BROWSER,
+          use: { browserName: browser },
+        }))),
+  ],
   webServer: {
     command: 'node e2e/prepare-workspace.mjs && bun ./dist/index.js',
     url: `http://127.0.0.1:${PORT}/healthz`,

apps/editor/src/lib/editor/create-editor.ts

@@ -49,6 +49,7 @@ import { registerCorePanels } from '$lib/project/register-core-panels';
 
 import { followProjectCode } from '../code/follow-project-code';
 import { followProjectPlugins, toDescriptor } from '../plugin/follow-project-plugins';
+import { followProjectCaches } from '../project/follow-project-caches';
 import { followProjectTrust } from '../project/follow-project-trust';
 import { followProjectRuntime } from '../runtime/follow-project-runtime';
 import { WorkspaceActions, WorkspaceActionsToken } from '../workspace/workspace-actions';
@@ -136,6 +137,7 @@ export const createEditor = async (
   );
   const { settings } = settingsSetup;
   services.provide(AccountSyncToken, settingsSetup.account);
+  store.add(followProjectCaches({ services, projects, settings, session }));
 
   const history = store.add(
     new HistoryService({

apps/editor/src/lib/project/follow-project-caches.ts

@@ -0,0 +1,60 @@
+import type { Container, Disposable } from '@nanoforge-dev/editor-kernel';
+import {
+  IndexedDbTreeCache,
+  OpfsContentCache,
+  ProjectCaches,
+  ProjectCachesToken,
+  type ProjectService,
+  removeOtherUsers,
+  removeStaleOpfs,
+} from '@nanoforge-dev/editor-project';
+import type { SessionInfo } from '@nanoforge-dev/editor-protocol';
+import { CoreSettings, type SettingsService } from '@nanoforge-dev/editor-settings';
+
+/** Upkeep runs this long after a project opens, when the start is over. */
+const SWEEP_DELAY_MS = 15_000;
+
+export interface FollowProjectCachesOptions {
+  readonly services: Container;
+  readonly projects: ProjectService;
+  readonly settings: SettingsService;
+  readonly session: SessionInfo;
+}
+
+/**
+ * Keeps the files of opened projects in the browser (OPFS and IndexedDB), per user, within the
+ * Storage settings. A hosted editor removes what other users left on this browser (a shared
+ * computer). Without the storage APIs, files are only kept while the page lives.
+ */
+export const followProjectCaches = ({
+  services,
+  projects,
+  settings,
+  session,
+}: FollowProjectCachesOptions): Disposable => {
+  if (!OpfsContentCache.isSupported() || !IndexedDbTreeCache.isSupported())
+    return { dispose: () => undefined };
+  const user = session.user?.id ?? 'local';
+  const online = session.mode === 'ONLINE';
+  void removeStaleOpfs(online ? user : undefined);
+  if (online) void removeOtherUsers(user);
+  const caches = new ProjectCaches(user, new OpfsContentCache(user), {
+    budgetBytes: () => settings.get(CoreSettings.cacheBudget) * 1024 * 1024,
+    maxAgeDays: () => settings.get(CoreSettings.cacheMaxAge),
+  });
+  projects.useCaches(caches);
+  services.provide(ProjectCachesToken, caches);
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const unsubscribe = projects.current.subscribe((project) => {
+    clearTimeout(timer);
+    if (project)
+      timer = setTimeout(() => void caches.sweep().catch(() => undefined), SWEEP_DELAY_MS);
+  });
+  return {
+    dispose: () => {
+      clearTimeout(timer);
+      unsubscribe();
+      caches.dispose();
+    },
+  };
+};

apps/editor/src/routes/(editor)/+page.svelte

@@ -12,6 +12,8 @@
     Switch,
   } from '@nanoforge-dev/editor-ui';
 
+  import { ProjectCachesToken } from '@nanoforge-dev/editor-project';
+
   import { getEditor } from '$lib/editor/editor-context';
 
   const editor = getEditor();
@@ -81,6 +83,7 @@
     try {
       await projectsApi.forget({ id: project.id });
       recent = recent?.filter((candidate) => candidate.id !== project.id);
+      void editor.services.tryGet(ProjectCachesToken)?.clearProject(project.id);
     } catch (reason) {
       error = reason instanceof Error ? reason.message : String(reason);
     }

docs/docs/editor/settings.mdx

@@ -25,6 +25,14 @@ When several scopes have a value, the more specific one wins: Project (only me),
 
 _Edit as JSON_ opens the project's settings file in the code editor, with completion.
 
+## Files kept in the browser
+
+The editor keeps the files of the projects you open in your browser, so opening one again downloads only what changed. A file that several projects or branches have is kept once.
+
+_Editor › Storage_ shows the space each project takes, with **Clear** for one project and **Clear all**. **Cache size** (500 MB) and **Forget projects after** (30 days) are saved on this machine: the least recently used files go first, and the editor never takes more than a tenth of what the browser allows. Removing a project from _Recent_ also clears its files. Clearing only drops these copies: unsaved changes and undo history are kept apart and never cleared. Switch on **Keep these files when the browser runs short of space** so the browser does not remove them on its own.
+
+In the online editor, signing in as someone else on the same browser removes the previous user's files.
+
 ## Import and export
 
 The _Import & export_ page saves a scope to a `.json` file, and loads one with a preview of what would change.

packages/code/src/service/code-service.ts

@@ -324,6 +324,14 @@ export class CodeService implements Disposable {
     const paths = this._options.fs.entries.filter(
       (entry) => entry.kind === 'file' && isCodeFile(entry.path),
     );
+    /** Fills the cache in a few requests; the reads below then cost nothing. */
+    await this._options.fs
+      .readMany(
+        paths
+          .filter((entry) => !this._options.documents.get(entry.path))
+          .map((entry) => entry.path),
+      )
+      .catch(() => undefined);
     for (let i = 0; i < paths.length; i += 32) {
       const batch = await Promise.all(
         paths.slice(i, i + 32).map(async (entry) => ({

packages/project/package.json

@@ -36,7 +36,7 @@
     "lint": "prettier --check . && eslint --format=pretty .",
     "format": "prettier --write . && eslint --fix --format=pretty src",
     "typecheck": "tsc -p tsconfig.json",
-    "test:unit": "vitest run --config ../../vitest.config.ts"
+    "test:unit": "vitest run --config vitest.config.ts"
   },
   "dependencies": {
     "@nanoforge-dev/editor-kernel": "workspace:*",
@@ -46,6 +46,8 @@
   },
   "devDependencies": {
     "@nanoforge-dev/editor-server-core": "workspace:*",
+    "@vitest/browser-playwright": "catalog:test",
+    "playwright": "catalog:test",
     "typescript": "catalog:build",
     "vitest": "catalog:test"
   },