From 4484d29bbe0f3af7d6ab76b7b7854f0981b2c7e0 Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Mon, 19 Oct 2026 02:24:23 +0200 Subject: [PATCH 1/2] Reuse highlighted code across builds through a shared persistent cache. --- .../guides/extending/plugins.mdx | 20 +- .../guides/getting-started/purestack-cli.mdx | 7 +- .../guides/getting-started/site-config.mdx | 6 +- packages/purestack/README.md | 2 + packages/ts-common/src/code-highlighter.ts | 2 + packages/ts-common/src/site-config-types.ts | 9 +- packages/ts-components/package.json | 2 +- packages/ts-highlight/README.md | 18 +- packages/ts-highlight/package.json | 1 + .../ts-highlight/src/codeLanguages.test.ts | 16 +- packages/ts-highlight/src/index.ts | 18 +- packages/ts-highlight/src/plugin.test.ts | 43 +- packages/ts-highlight/src/shiki.test.ts | 80 ++++ packages/ts-highlight/src/shiki.ts | 49 +- packages/ts-highlight/src/shikiCache.ts | 22 + packages/ts-images/README.md | 3 + packages/ts-images/src/index.ts | 10 +- packages/ts-images/src/processor.test.ts | 29 ++ packages/ts-images/src/processor.ts | 120 +++-- packages/ts-render/package.json | 2 +- packages/ts-ssg/README.md | 9 +- packages/ts-ssg/package.json | 2 +- packages/ts-ssg/src/build/build-config.ts | 9 +- .../ts-ssg/src/build/incremental/index.ts | 41 +- .../ts-ssg/src/build/incremental/support.ts | 3 +- packages/ts-ssg/src/build/manifest.ts | 4 +- packages/ts-ssg/src/build/site.ts | 2 + packages/ts-ssg/src/cli-runner.ts | 25 +- packages/ts-ssg/src/config/core-config.ts | 1 + packages/ts-ssg/src/index.ts | 1 + packages/ts-ssg/src/plugins/config.test.ts | 3 +- .../ts-ssg/src/plugins/diagnostics.test.ts | 2 +- .../src/plugins/highlighter-flush.test.ts | 184 ++++++++ packages/ts-ssg/src/plugins/plugin.ts | 23 +- packages/ts-ssg/src/plugins/types.ts | 8 +- packages/ts-util-node/README.md | 113 ++++- packages/ts-util-node/package.json | 4 + packages/ts-util-node/src/cacheStorage.ts | 194 ++++++++ packages/ts-util-node/src/index.ts | 5 + .../ts-util-node/src/persistentCache.test.ts | 430 ++++++++++++++++++ packages/ts-util-node/src/persistentCache.ts | 415 +++++++++++++++++ yarn.lock | 18 +- 42 files changed, 1817 insertions(+), 138 deletions(-) create mode 100644 packages/ts-highlight/src/shiki.test.ts create mode 100644 packages/ts-highlight/src/shikiCache.ts create mode 100644 packages/ts-ssg/src/plugins/highlighter-flush.test.ts create mode 100644 packages/ts-util-node/src/cacheStorage.ts create mode 100644 packages/ts-util-node/src/persistentCache.test.ts create mode 100644 packages/ts-util-node/src/persistentCache.ts diff --git a/frontend/purestack.studio/guides/extending/plugins.mdx b/frontend/purestack.studio/guides/extending/plugins.mdx index 5f335f44..4d98b04f 100644 --- a/frontend/purestack.studio/guides/extending/plugins.mdx +++ b/frontend/purestack.studio/guides/extending/plugins.mdx @@ -185,7 +185,7 @@ This keeps the layouts and components while omitting automatic social metadata, An explicitly listed standard plugin replaces the automatic instance by name and keeps its initialization position. You can also use `createDefaultPlugins()` to obtain the complete set. In programmatic builds, use `options.defaultPlugins` alongside `options.plugins`. -The highlighting plugin selects the configured backend. Configure native backend factories through `highlightPlugin({ createShiki, createHighlightJs })`, or supply `markdown.createHighlighter(config)` for a different engine. The generic provider returns `{ codeToHtml(code, lang?) }`. Ordinary providers override the default without constructing it; per-language `markdown.resolveHighlighter` callbacks retain precedence. +The highlighting plugin selects the configured backend. Configure native backend factories through `highlightPlugin({ createShiki, createHighlightJs })`, or supply `markdown.createHighlighter({ mdx, cache })` for a different engine. The generic provider returns `{ codeToHtml(code, lang?) }`. Ordinary providers override the default without constructing it; per-language `markdown.resolveHighlighter` callbacks retain precedence. The Pagefind plugin supplies a `search` provider. To use another engine, replace `purestack:pagefind` or disable the default set and install your own provider. The builder supplies excluded output paths and the rendered-page list for lazy requests, and manages invalidation on edits and deletions. Sitemap and robots generation remain built in. @@ -236,6 +236,8 @@ export const brandPlugin = definePlugin({ `resolveSiteConfig(input, { plugins, defaultPlugins })` composes the supplied plugin instances with the standard plugins by default. This synchronous helper does not apply JSON plugin selection (`plugins` or `defaultPlugins`) or load `purestack.config.ts`. Plugin-owned sections (`style`, `docLayout`, `pageToc`, `logo`, `auth`, `consent`, `analytics`, `pagefind`, and `preview`) are optional in `SiteConfig`: an omitted plugin does not supply defaults or validate its unused input. A replacement plugin reads its settings from `input` and `file`, then contributes normalized config. Analytics also resolves explicitly configured consent when its browser runtime is omitted, so enabled consent is never bypassed. Configuration contributions are synchronous and may run whenever a project configuration is resolved; keep file writes and other side effects in lifecycle hooks. +`config.cache` is `false` when a run uses `--no-cache` or `options.cache: false`. A plugin that keeps a persistent cache outside the output directory should neither read nor write it then. Site settings cannot change this flag, and it never invalidates incremental output. + Builder style state is isolated across asynchronous operations, including concurrent sites. Its owned theme options remain linked to `context.config.style.theme`, so in-place config-hook changes and `themes.setOptions` update page colors, CSS, and generated favicons consistently. Head, metadata, template, Markdown, provider, and lifecycle errors identify their owning plugin and preserve the original cause. Markdown wrappers preserve unified presets, duplicate-plugin option merging, and callback transforms. ## Optimize images @@ -617,6 +619,22 @@ When you publish the package: - **Ship JavaScript.** Packages load without bundling, so publish compiled JavaScript with type declarations, not TypeScript source. - **Share the site's PureStack.** List `purestack`, and any `@purestack/*` package the plugin imports, under `peerDependencies`, not `dependencies`. The plugin then uses the site's copy instead of installing its own. +## Cache build-time computations + +Plugins and build-time components can use `PersistentCache` from +`@purestack/ts-util-node` to reuse expensive results across builds. Create one +cache per factory, use an owner namespace and implementation version, and include +all output-affecting inputs in its keys. Pass `enabled: config.cache` to honor +`--no-cache`. + +Only cache computed values: dependency tracking, styles, and asset emission must +still run when a value is reused. The cache batches small values into JSON shards +and keeps bounded memory. Await `cache.flush()` after the computation batch; +periodic development flushing does not keep Node alive. Highlighters can return +an optional `flush()` method, which core awaits after each build or development +render batch, including failures. See the +[cache API and component example](https://github.com/PureStackStudio/PureStack/tree/main/packages/ts-util-node#persistent-build-cache). + ## Build from a script `buildSite` and `startDevServer` run the same build as the CLI, for a project that needs its own build script. They do not look for `purestack.config.ts`; pass plugins in `options.plugins`: diff --git a/frontend/purestack.studio/guides/getting-started/purestack-cli.mdx b/frontend/purestack.studio/guides/getting-started/purestack-cli.mdx index 82c03226..7dcd8ea2 100644 --- a/frontend/purestack.studio/guides/getting-started/purestack-cli.mdx +++ b/frontend/purestack.studio/guides/getting-started/purestack-cli.mdx @@ -115,6 +115,10 @@ yarn purestack build --content ./content --clean `--clean` removes the configured `outDir` before rebuilding it. Use it when you need an output folder containing only files from the current build. Without `--clean`, PureStack keeps the directory and rewrites current outputs. +`--no-cache` bypasses reads and writes to persistent computation caches kept outside the output directory, such as highlighted code and optimized images. It works with `build`, `serve`, and `publish`. Use it to compare uncached rendering or investigate a stale cached result. Existing cache files stay in place; a cold build with caching enabled also spends time populating its cache. + +During `serve`, pages still render on request and reuse their rendered output until invalidated. `--no-cache` affects computations when a page renders; it does not force rendering on every request. To rebuild into an empty output directory with persistent caches bypassed, combine `--clean` and `--no-cache` (`publish` already cleans its output directory). + ### `serve`: preview and edit ```sh @@ -136,6 +140,7 @@ These options belong to `serve`: | `--host ` | Listen on a particular interface, for example `127.0.0.1`. | | `--port ` | Listen on a different port. | | `--clean` | Remove `outDir` before setup and when config changes reset shared setup. | +| `--no-cache` | Skip persistent caches, such as highlighted code and optimized images. | | `--full-render` | Build every page, asset, and the complete search index once at startup. Watched edits remain incremental. | | `--no-watch` | Keep the server running without watching the content directory. Pages still render on request, or all at startup with `--full-render`. | | `--no-reload` | Keep watching and updating the site, but do not inject the browser live reload script. | @@ -150,7 +155,7 @@ yarn purestack publish --content ./content `publish` uses `publishDir` instead of `outDir`. It cleans that directory before building, minifies generated HTML and browser scripts, disables pretty printed CSS, and treats asset build errors as failures. It creates the same site content and optional search and sitemap outputs as `build`, according to `siteConfig.json`. -Point your deployment process at `publishDir` after a successful run. `publish` prepares files locally; it does not upload them. It accepts `--content`, with no separate `--clean` flag because cleaning is built in. +Point your deployment process at `publishDir` after a successful run. `publish` prepares files locally; it does not upload them. It accepts `--content` and `--no-cache`, with no separate `--clean` flag because cleaning is built in. ## Choose the public URL diff --git a/frontend/purestack.studio/guides/getting-started/site-config.mdx b/frontend/purestack.studio/guides/getting-started/site-config.mdx index 8e3d8363..69b5070e 100644 --- a/frontend/purestack.studio/guides/getting-started/site-config.mdx +++ b/frontend/purestack.studio/guides/getting-started/site-config.mdx @@ -190,7 +190,11 @@ Place this in `content/guides/_nav.json`. A custom item's `url` is relative to t } ``` -`mdx.highlighter` accepts `highlightjs` (the default and faster option) or `shiki`. `disableHighlighter` turns highlighting off. `compileMdAsMdx` defaults to `true`, so `.md` files use the MDX/Regor markup pipeline; `.mdx` files use that pipeline in either case. The [Regor guide](/guides/extending/regor/) explains the supported markup and components. +`mdx.highlighter` accepts `highlightjs` (the default) or `shiki`. The standard Shiki backend caches highlighted HTML across builds in the OS temporary directory. Cache keys include the code, language, themes, loaded languages, and Shiki version. On a warm build, cached blocks reuse their HTML, and Shiki initializes only when a block needs highlighting. Output from a custom `createShiki` factory is not cached. + +Use the CLI's [`--no-cache` flag](/guides/getting-started/purestack-cli/#build-generate-static-files) to bypass persistent caches for a run. This is a run option, not a `siteConfig.json` setting. + +`disableHighlighter` turns highlighting off. `compileMdAsMdx` defaults to `true`, so `.md` files use the MDX/Regor markup pipeline; `.mdx` files use that pipeline in either case. The [Regor guide](/guides/extending/regor/) explains the supported markup and components. ## Search diff --git a/packages/purestack/README.md b/packages/purestack/README.md index 7a7c5d19..7b8d316f 100644 --- a/packages/purestack/README.md +++ b/packages/purestack/README.md @@ -103,6 +103,8 @@ Run these commands from the project root: `publish` prepares files for a static host; it does not upload them. Use `--clean` with `build` or `serve` to clear the output directory before a build. +Use `--no-cache` with any command to skip persistent caches, such as +highlighted code and optimized images, for a fully cold build. Use `--host` and `--port` with `serve` to choose its address. Every command needs `--content` pointing to a directory with `siteConfig.json`. diff --git a/packages/ts-common/src/code-highlighter.ts b/packages/ts-common/src/code-highlighter.ts index af7aee88..0b753827 100644 --- a/packages/ts-common/src/code-highlighter.ts +++ b/packages/ts-common/src/code-highlighter.ts @@ -1,4 +1,6 @@ /** Converts source code to highlighted pre/code HTML. */ export interface MdxCodeHighlighter { codeToHtml: (code: string, lang?: string) => string | Promise + /** Persist buffered computation results after a render batch. Await producers before calling. */ + flush?: () => void | Promise } diff --git a/packages/ts-common/src/site-config-types.ts b/packages/ts-common/src/site-config-types.ts index e773ae20..4a3a38d0 100644 --- a/packages/ts-common/src/site-config-types.ts +++ b/packages/ts-common/src/site-config-types.ts @@ -22,6 +22,11 @@ export interface CoreSiteConfig { sitemap: SitemapConfig i18n: I18nConfig mdx: SiteMdxConfig + /** + * Whether plugins may read and write persistent caches outside the output + * directory. Set per run (`--no-cache`, `options.cache`), never by site settings. + */ + cache: boolean } export interface SiteConfig extends CoreSiteConfig { @@ -156,7 +161,9 @@ export type SiteStyleConfigInput = DeepPartial< theme?: ThemeOptionsInput } -export type SiteConfigInput = DeepPartial> & { +export type SiteConfigInput = DeepPartial< + Omit +> & { style?: SiteStyleConfigInput } diff --git a/packages/ts-components/package.json b/packages/ts-components/package.json index 35f03586..70723797 100644 --- a/packages/ts-components/package.json +++ b/packages/ts-components/package.json @@ -56,7 +56,7 @@ "@purestack/ts-style": "workspace:^", "@purestack/ts-svg-icons": "workspace:^", "@purestack/ts-util": "workspace:^", - "regor": "^1.7.5" + "regor": "^1.7.6" }, "regorComponents": [ "AlertBox", diff --git a/packages/ts-highlight/README.md b/packages/ts-highlight/README.md index b46c634f..b228a27a 100644 --- a/packages/ts-highlight/README.md +++ b/packages/ts-highlight/README.md @@ -26,7 +26,23 @@ unused factory is never called. The Shiki options contain the default languages and themes. For highlight.js, prefer a new instance with explicitly registered grammars to avoid changing the shared registry. -`createMdxHighlighter`, `createHljsHighlighter`, `DEFAULT_MDX_CODE_LANGS`, and +The default Shiki backend caches highlighted HTML in the OS temporary +directory (`purestack-cache/shiki`), keyed by Shiki's version, the themes, the loaded languages, and the +code. Repeat builds reuse it, and Shiki loads only when a block is not cached. +Builds run with `--no-cache` skip it. Output from a custom `createShiki` +factory is never cached, because its grammars are unknown. + +Storage uses `PersistentCache` from `@purestack/ts-util-node`: 32 lazy JSON shards, +buffered writes merged under shard locks, bounded memory, and cleanup of entries +unused for 30 days. Large entries stay in separate files. PureStack flushes after +build and development render batches. Standalone users should await +`highlighter.flush?.()` after rendering their batch; the periodic flush timer +does not keep Node alive. +Other build-time plugins and components can use the same utility with their own +namespace, version, and input keys. + +`createMdxHighlighter(themes?, langs?, { create?, cache? })`, +`createHljsHighlighter`, `DEFAULT_MDX_CODE_LANGS`, and `DEFAULT_MDX_CODE_THEMES` are exported for standalone use. The shared `MdxCodeHighlighter` contract is exported by `ts-common` and `ts-ssg`. diff --git a/packages/ts-highlight/package.json b/packages/ts-highlight/package.json index fbf0b247..c517e2cc 100644 --- a/packages/ts-highlight/package.json +++ b/packages/ts-highlight/package.json @@ -40,6 +40,7 @@ }, "dependencies": { "@purestack/ts-common": "workspace:^", + "@purestack/ts-util-node": "workspace:^", "highlight.js": "^11.11.1", "shiki": "^3.22.0" } diff --git a/packages/ts-highlight/src/codeLanguages.test.ts b/packages/ts-highlight/src/codeLanguages.test.ts index 2f96b270..2a908ac8 100644 --- a/packages/ts-highlight/src/codeLanguages.test.ts +++ b/packages/ts-highlight/src/codeLanguages.test.ts @@ -22,11 +22,9 @@ describe('MDX code languages', () => { }) it('renders every inferred language with the default Shiki grammars', async () => { - const highlighter = await createMdxHighlighter( - undefined, - undefined, - createTestShiki, - ) + const highlighter = await createMdxHighlighter(undefined, undefined, { + create: createTestShiki, + }) const languages = new Set([ ...DEFAULT_MDX_CODE_LANGS, ...Object.values(LANGUAGE_BY_EXTENSION), @@ -65,11 +63,9 @@ describe('MDX code languages', () => { ['gql', 'graphql', 'query { user { id } }'], ])('resolves %s with both highlighters', async (alias, language, code) => { // Loading by the alias must also work when defaults are overridden. - const shiki = await createMdxHighlighter( - undefined, - [alias], - createTestShiki, - ) + const shiki = await createMdxHighlighter(undefined, [alias], { + create: createTestShiki, + }) expect(await shiki.codeToHtml(code, alias)).toBe( await shiki.codeToHtml(code, language), ) diff --git a/packages/ts-highlight/src/index.ts b/packages/ts-highlight/src/index.ts index 3497e428..7c433faf 100644 --- a/packages/ts-highlight/src/index.ts +++ b/packages/ts-highlight/src/index.ts @@ -10,6 +10,7 @@ export { DEFAULT_MDX_CODE_THEMES, type MdxCodeLangs, type MdxCodeThemes, + type MdxHighlighterOptions, type MdxShikiFactory, } from './shiki' @@ -31,13 +32,22 @@ export function highlightPlugin(options: HighlightPluginOptions = {}) { }, }, markdown: { - async createHighlighter(config: Partial) { - if (config.disableHighlighter) return undefined - const backend = backendIdentifier(config.highlighter) + async createHighlighter({ + mdx, + cache, + }: { + mdx: Partial + cache: boolean + }) { + if (mdx.disableHighlighter) return undefined + const backend = backendIdentifier(mdx.highlighter) if (backend === 'highlightjs') return createHljsHighlighter(await options.createHighlightJs?.()) if (backend === 'shiki') - return createMdxHighlighter(undefined, undefined, options.createShiki) + return createMdxHighlighter(undefined, undefined, { + create: options.createShiki, + cache, + }) throw new Error( `The standard highlighting plugin does not support backend "${backend}". Supply a custom markdown.createHighlighter provider.`, ) diff --git a/packages/ts-highlight/src/plugin.test.ts b/packages/ts-highlight/src/plugin.test.ts index 45573674..ba815533 100644 --- a/packages/ts-highlight/src/plugin.test.ts +++ b/packages/ts-highlight/src/plugin.test.ts @@ -11,15 +11,40 @@ import { highlightPlugin } from './index' describe('highlightPlugin', () => { it('owns its default selection and validates identifiers only when its backend is used', async () => { const plugin = highlightPlugin() - const fallback = await plugin.markdown.createHighlighter({}) + const fallback = await plugin.markdown.createHighlighter({ + mdx: {}, + cache: true, + }) expect(fallback?.codeToHtml('const answer = 42', 'ts')).toContain( 'hljs-keyword', ) await expect( - plugin.markdown.createHighlighter({ highlighter: 'another-provider' }), + plugin.markdown.createHighlighter({ + mdx: { highlighter: 'another-provider' }, + cache: true, + }), ).rejects.toThrow('Supply a custom markdown.createHighlighter provider') }) + it('highlights Shiki code identically with and without the persistent cache', async () => { + const code = 'const view = html`

Saved

`' + const render = async (cache: boolean) => { + const highlighter = await highlightPlugin().markdown.createHighlighter({ + mdx: { highlighter: 'shiki' }, + cache, + }) + try { + return await highlighter?.codeToHtml(code, 'ts') + } finally { + await highlighter?.flush?.() + } + } + const uncached = await render(false) + expect(uncached).toContain('--shiki-light:') + expect(await render(true)).toBe(uncached) + expect(await render(true)).toBe(uncached) + }) + it.each(['sync', 'async'])( 'uses a %s Shiki factory with default options and a custom grammar', async (mode) => { @@ -59,7 +84,8 @@ describe('highlightPlugin', () => { }, }) const highlighter = await plugin.markdown.createHighlighter({ - highlighter: 'shiki', + mdx: { highlighter: 'shiki' }, + cache: true, }) expect(calls).toBe(1) expect( @@ -99,7 +125,8 @@ describe('highlightPlugin', () => { }, }) const highlighter = await plugin.markdown.createHighlighter({ - highlighter: 'highlightjs', + mdx: { highlighter: 'highlightjs' }, + cache: true, }) expect(calls).toBe(1) expect(highlighter?.codeToHtml('launch', 'creator-test-alias')).toContain( @@ -110,7 +137,8 @@ describe('highlightPlugin', () => { ) expect(hljs.getLanguage('creator-test')).toBeUndefined() const defaults = await highlightPlugin().markdown.createHighlighter({ - highlighter: 'highlightjs', + mdx: { highlighter: 'highlightjs' }, + cache: true, }) expect(defaults?.codeToHtml('const answer = 42', 'ts')).toContain( 'hljs-keyword', @@ -127,7 +155,10 @@ describe('highlightPlugin', () => { createHighlightJs: fail, }) expect( - await plugin.markdown.createHighlighter({ disableHighlighter: true }), + await plugin.markdown.createHighlighter({ + mdx: { disableHighlighter: true }, + cache: true, + }), ).toBeUndefined() }) }) diff --git a/packages/ts-highlight/src/shiki.test.ts b/packages/ts-highlight/src/shiki.test.ts new file mode 100644 index 00000000..51ccbb51 --- /dev/null +++ b/packages/ts-highlight/src/shiki.test.ts @@ -0,0 +1,80 @@ +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import type { Highlighter } from 'shiki' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createMdxHighlighter } from './shiki' + +const mocks = vi.hoisted(() => ({ directory: '', create: vi.fn() })) +vi.mock('shiki', () => ({ createHighlighter: mocks.create })) +vi.mock('node:os', async (original) => { + const module = await original() + mocks.directory = module.tmpdir() + return { + ...module, + default: { ...module, tmpdir: () => mocks.directory }, + } +}) + +describe('Shiki cache integration', () => { + let directory: string + let systemTemporary: string + + beforeEach(async () => { + systemTemporary = os.tmpdir() + directory = await fs.mkdtemp( + path.join(systemTemporary, 'purestack-shiki-test-'), + ) + mocks.directory = directory + mocks.create.mockReset() + mocks.create.mockResolvedValue({ + codeToHtml: (code: string) => `
${code}
`, + }) + }) + + afterEach(async () => { + mocks.directory = systemTemporary + await fs.rm(directory, { recursive: true, force: true }) + }) + + it('loads Shiki once on a cold miss and reuses disk output without loading it in a new instance', async () => { + const cold = await createMdxHighlighter() + expect(mocks.create).not.toHaveBeenCalled() + const results = await Promise.all( + Array.from({ length: 10 }, () => + cold.codeToHtml('const answer = 42', 'ts'), + ), + ) + expect(results).toEqual(Array(10).fill('
const answer = 42
')) + expect(mocks.create).toHaveBeenCalledTimes(1) + await cold.flush?.() + mocks.create.mockRejectedValue( + new Error('Warm builds must not initialize Shiki'), + ) + const warm = await createMdxHighlighter() + expect(await warm.codeToHtml('const answer = 42', 'typescript')).toBe( + results[0], + ) + expect(mocks.create).toHaveBeenCalledTimes(1) + await warm.flush?.() + }) + + it('bypasses persistence for disabled caching and custom grammars', async () => { + const uncached = await createMdxHighlighter(undefined, undefined, { + cache: false, + }) + expect(await uncached.codeToHtml('uncached', 'ts')).toBe( + '
uncached
', + ) + const custom = await createMdxHighlighter(undefined, undefined, { + create: () => + ({ + codeToHtml: () => '
custom grammar
', + }) as unknown as Highlighter, + }) + expect(await custom.codeToHtml('custom', 'unknown-grammar')).toBe( + '
custom grammar
', + ) + expect(await fs.readdir(directory)).toEqual([]) + }) +}) diff --git a/packages/ts-highlight/src/shiki.ts b/packages/ts-highlight/src/shiki.ts index 1cd411e7..0c468938 100644 --- a/packages/ts-highlight/src/shiki.ts +++ b/packages/ts-highlight/src/shiki.ts @@ -4,6 +4,7 @@ import { normalizeCodeLanguage, } from '@purestack/ts-common' import { createHighlighter, type Highlighter } from 'shiki' +import { openShikiCache } from './shikiCache' export type MdxShikiFactory = ( options: Parameters[0], @@ -25,24 +26,48 @@ export const DEFAULT_MDX_CODE_LANGS: MdxCodeLangs = DEFAULT_CODE_LANGUAGES const TYPESCRIPT_WITH_TAGS_LANG = 'ts-tags' +export interface MdxHighlighterOptions { + /** A custom factory; its grammars are unknown, so its output is never cached. */ + create?: MdxShikiFactory + /** `false` disables the persistent cache for this run. Defaults to `true`. */ + cache?: boolean +} + export async function createMdxHighlighter( themes: MdxCodeThemes = DEFAULT_MDX_CODE_THEMES, langs: MdxCodeLangs = DEFAULT_MDX_CODE_LANGS, - create: MdxShikiFactory = createHighlighter, + { create, cache: useCache = true }: MdxHighlighterOptions = {}, ): Promise { - const themeList = dedupe([themes.light, themes.dark]) - const langList = resolveLoadedLanguages(langs) - const highlighter = await create({ - themes: themeList, - langs: langList, - }) + const options = { + themes: dedupe([themes.light, themes.dark]), + langs: resolveLoadedLanguages(langs), + } + const render = (highlighter: Highlighter, code: string, lang: string) => + highlighter.codeToHtml(code, { + lang, + themes: { light: themes.light, dark: themes.dark }, + defaultColor: false, + }) + + if (create || !useCache) { + const highlighter = await (create ?? createHighlighter)(options) + return { + codeToHtml: (code: string, lang?: string) => + render(highlighter, code, resolveHighlightLanguage(lang)), + } + } + // Bundled grammars and themes are identified by name and Shiki's version, so + // their output is reused across builds. Shiki loads on the first cache miss. + const cache = openShikiCache(themes, options.langs) + let highlighter: Promise | undefined return { - codeToHtml: (code: string, lang?: string) => { - return highlighter.codeToHtml(code, { - lang: resolveHighlightLanguage(lang), - themes: { light: themes.light, dark: themes.dark }, - defaultColor: false, + flush: () => cache.flush(), + async codeToHtml(code: string, lang?: string) { + const resolved = resolveHighlightLanguage(lang) + return cache.getOrSetAsync(`${resolved}\0${code}`, async () => { + highlighter ??= createHighlighter(options) + return render(await highlighter, code, resolved) }) }, } diff --git a/packages/ts-highlight/src/shikiCache.ts b/packages/ts-highlight/src/shikiCache.ts new file mode 100644 index 00000000..440061bf --- /dev/null +++ b/packages/ts-highlight/src/shikiCache.ts @@ -0,0 +1,22 @@ +import { createRequire } from 'node:module' +import { PersistentCache } from '@purestack/ts-util-node' + +// Bump when the highlighter's rendering options change cached output. +const CACHE_SCHEMA = 1 +const SHIKI_VERSION: string = createRequire(import.meta.url)( + 'shiki/package.json', +).version + +export function openShikiCache( + themes: Record<'light' | 'dark', string>, + languages: readonly string[], +) { + return new PersistentCache({ + namespace: 'shiki', + version: JSON.stringify([CACHE_SCHEMA, SHIKI_VERSION, themes, languages]), + codec: { + encode: (html) => html, + decode: (data) => data.toString('utf8'), + }, + }) +} diff --git a/packages/ts-images/README.md b/packages/ts-images/README.md index 70d7ba26..5cc878b1 100644 --- a/packages/ts-images/README.md +++ b/packages/ts-images/README.md @@ -152,6 +152,9 @@ to the discovered route/asset inventory may also invalidate other pages under the normal URL-resolution rules. Dependencies survive manifest reuse and missing images remain tracked for error-page recovery. +Runs with `--no-cache` (or `options.cache: false`) encode every image and +neither read nor write the cache. + The persistent cache survives cleaned output directories. Keys include input bytes, pixel settings, and codec versions; markup-only choices such as `alt`, loading, and `sizes` reuse encoded files. Cache contents are verified by hash, diff --git a/packages/ts-images/src/index.ts b/packages/ts-images/src/index.ts index 51d640bf..183f07a9 100644 --- a/packages/ts-images/src/index.ts +++ b/packages/ts-images/src/index.ts @@ -64,11 +64,11 @@ export function imagesPlugin(options: ImagesPluginOptions = {}) { if (!config) throw new Error('Images requires resolved extensions.images settings.') if (!config.enabled) return undefined - const processor = new ImageProcessor(config, [ - site.contentDir, - site.outDir, - site.publishDir, - ]) + const processor = new ImageProcessor( + config, + [site.contentDir, site.outDir, site.publishDir], + site.cache, + ) return { namespace: 'images', async processPage(input) { diff --git a/packages/ts-images/src/processor.test.ts b/packages/ts-images/src/processor.test.ts index ca96f506..dbf9fc47 100644 --- a/packages/ts-images/src/processor.test.ts +++ b/packages/ts-images/src/processor.test.ts @@ -200,6 +200,35 @@ describe('image processor and persistent cache', () => { expect(imagePreset(settings, 'screenshot').lossless).toBe(true) }) + it('encodes identical bytes without touching the cache when caching is off', async () => { + await sharp({ + create: { width: 64, height: 32, channels: 3, background: 'teal' }, + }) + .png() + .toFile(source().absPath) + const settings = config() + const uncached = await new ImageProcessor(settings, [], false).process( + source(), + settings, + ) + expect(uncached?.cache).toBe('miss') + await expect(fs.stat(settings.cacheDir)).rejects.toMatchObject({ + code: 'ENOENT', + }) + const cached = await new ImageProcessor(settings).process( + source(), + settings, + ) + expect(cached?.variants.map((variant) => variant.hash)).toEqual( + uncached?.variants.map((variant) => variant.hash), + ) + const again = await new ImageProcessor(settings, [], false).process( + source(), + settings, + ) + expect(again?.cache).toBe('miss') + }) + it('rejects input byte and pixel limits with the source path', async () => { await sharp({ create: { width: 64, height: 32, channels: 3, background: 'green' }, diff --git a/packages/ts-images/src/processor.ts b/packages/ts-images/src/processor.ts index 15506a98..d4c148ff 100644 --- a/packages/ts-images/src/processor.ts +++ b/packages/ts-images/src/processor.ts @@ -70,6 +70,8 @@ export class ImageProcessor { constructor( private readonly config: ImagesConfig, private readonly protectedDirectories: readonly string[] = [], + /** `false` encodes every image without reading or writing the cache. */ + private readonly useCache = true, ) { this.queue = new WorkQueue(config.concurrency) } @@ -184,6 +186,24 @@ export class ImageProcessor { fallback: OutputFormat, preset: ResolvedImagePreset, ): Promise { + const expected = widths.flatMap((outputWidth) => + formats.map((format) => ({ + width: outputWidth, + height: Math.max(1, Math.round((height * outputWidth) / width)), + format, + file: fileName(outputWidth, format), + })), + ) + if (!this.useCache) + return { + key, + width, + height, + fallback, + variants: await this.encode(sharp, data, expected, preset), + sourceBytes: data.length, + cache: 'miss', + } this.cacheRoot ??= this.prepareCache().catch((error) => { this.cacheRoot = undefined throw error @@ -209,14 +229,6 @@ export class ImageProcessor { const directoryStat = await optionalStat(directory) if (directoryStat?.isSymbolicLink()) throw new Error('Image cache entries must not be symlinks.') - const expected = widths.flatMap((outputWidth) => - formats.map((format) => ({ - width: outputWidth, - height: Math.max(1, Math.round((height * outputWidth) / width)), - format, - file: fileName(outputWidth, format), - })), - ) const cached = await readCache(directory, key, expected) if (cached) { const now = new Date() @@ -235,47 +247,7 @@ export class ImageProcessor { cache: 'hit', } } - const variants: ImageVariant[] = [] - for (const item of expected) { - let pipeline = sharp(data, { - limitInputPixels: this.config.maxInputPixels, - failOn: 'warning', - }) - .autoOrient() - .resize({ width: item.width, withoutEnlargement: true }) - .toColorspace('srgb') - if (preset.metadata === 'preserve') - pipeline = pipeline.withMetadata({ orientation: 1 }) - if (item.format === 'avif') - pipeline = pipeline.avif({ - quality: preset.quality.avif, - lossless: preset.lossless, - effort: 4, - }) - else if (item.format === 'webp') - pipeline = pipeline.webp({ - quality: preset.quality.webp, - lossless: preset.lossless, - effort: 4, - }) - else if (item.format === 'jpeg') - pipeline = pipeline.jpeg({ - quality: preset.quality.jpeg, - mozjpeg: true, - }) - else pipeline = pipeline.png({ compressionLevel: 9, palette: false }) - const result = await pipeline - .timeout({ seconds: 30 }) - .toBuffer({ resolveWithObject: true }) - variants.push({ - file: item.file, - format: item.format, - width: result.info.width, - height: result.info.height, - data: result.data, - hash: digest(result.data), - }) - } + const variants = await this.encode(sharp, data, expected, preset) const manifest: CacheManifest = { version: 1, key, @@ -344,6 +316,56 @@ export class ImageProcessor { } } + private async encode( + sharp: typeof sharpType, + data: Buffer, + expected: Array<{ file: string; width: number; format: OutputFormat }>, + preset: ResolvedImagePreset, + ): Promise { + const variants: ImageVariant[] = [] + for (const item of expected) { + let pipeline = sharp(data, { + limitInputPixels: this.config.maxInputPixels, + failOn: 'warning', + }) + .autoOrient() + .resize({ width: item.width, withoutEnlargement: true }) + .toColorspace('srgb') + if (preset.metadata === 'preserve') + pipeline = pipeline.withMetadata({ orientation: 1 }) + if (item.format === 'avif') + pipeline = pipeline.avif({ + quality: preset.quality.avif, + lossless: preset.lossless, + effort: 4, + }) + else if (item.format === 'webp') + pipeline = pipeline.webp({ + quality: preset.quality.webp, + lossless: preset.lossless, + effort: 4, + }) + else if (item.format === 'jpeg') + pipeline = pipeline.jpeg({ + quality: preset.quality.jpeg, + mozjpeg: true, + }) + else pipeline = pipeline.png({ compressionLevel: 9, palette: false }) + const result = await pipeline + .timeout({ seconds: 30 }) + .toBuffer({ resolveWithObject: true }) + variants.push({ + file: item.file, + format: item.format, + width: result.info.width, + height: result.info.height, + data: result.data, + hash: digest(result.data), + }) + } + return variants + } + private async prepareCache() { const directory = await canonicalPath(this.config.cacheDir) for (const protectedDirectory of this.protectedDirectories) { diff --git a/packages/ts-render/package.json b/packages/ts-render/package.json index e359a95a..72f7d7a6 100644 --- a/packages/ts-render/package.json +++ b/packages/ts-render/package.json @@ -42,6 +42,6 @@ "@purestack/ts-common": "workspace:^", "@purestack/ts-minidom": "workspace:^", "@purestack/ts-util": "workspace:^", - "regor": "^1.7.5" + "regor": "^1.7.6" } } diff --git a/packages/ts-ssg/README.md b/packages/ts-ssg/README.md index 4af47238..9b67a782 100644 --- a/packages/ts-ssg/README.md +++ b/packages/ts-ssg/README.md @@ -67,7 +67,8 @@ await startDevServer({ Flags: - `--content ./content` or `--content=./content` (required; directory must contain `siteConfig.json`) -- `--clean` (`build`, `serve`) +- `--clean` (`build`, `serve`; empty the output directory and ignore its incremental manifest) +- `--no-cache` (`build`, `serve`, `publish`; skip persistent plugin caches outside the output directory, such as highlighted code and optimized images) - `--port 4173` or `--port=4173` (`serve`) - `--host 127.0.0.1` or `--host=127.0.0.1` (`serve`) - `--no-watch` (`serve`) @@ -530,6 +531,8 @@ export const brandPlugin = definePlugin({ `resolveSiteConfig(input, { plugins, defaultPlugins })` composes the supplied plugin instances with the standard plugins by default. Plugin-owned sections (`style`, `docLayout`, `pageToc`, `logo`, `auth`, `consent`, `analytics`, `pagefind`, and `preview`) are optional in `SiteConfig`: an omitted plugin does not supply defaults or validate its unused input. A replacement plugin reads its settings from `input` and `file`, then contributes normalized config. Analytics also resolves explicitly configured consent when its browser runtime is omitted, so enabled consent is never bypassed. Configuration contributions are synchronous and may run whenever a project configuration is resolved; keep file writes and other side effects in lifecycle hooks. +`config.cache` is `false` when a run uses `--no-cache` or `options.cache: false`. A plugin that keeps a persistent cache outside the output directory should neither read nor write it then. Site settings cannot change this flag, and it never invalidates incremental output. + Builder style state is isolated across asynchronous operations, including concurrent sites. Its owned theme options remain linked to `context.config.style.theme`, so in-place config-hook changes and `themes.setOptions` update page colors, CSS, and generated favicons consistently. Head, metadata, template, Markdown, provider, and lifecycle errors identify their owning plugin and preserve the original cause. Markdown wrappers preserve unified presets, duplicate-plugin option merging, and callback transforms. ### Code highlighting @@ -576,7 +579,9 @@ export default defineConfig({ The same plugin accepts `createHighlightJs()`, which returns a configured highlight.js instance synchronously or asynchronously. Use `hljs.newInstance()` and register the grammars you need to avoid mutating the shared registry. Backend-specific factories belong to `HighlightPluginOptions`; the SSG plugin API uses the backend-independent `markdown.createHighlighter(config)` contract. -A custom `markdown.createHighlighter` receives the MDX configuration and returns an object with `codeToHtml(code, lang?)`, or `undefined` to leave source unhighlighted. An ordinary provider replaces the default without initializing it. Competing providers in the same precedence group fail before either initializes. Per-language `resolveHighlighter(lang)` callbacks still run before the selected provider. +A custom `markdown.createHighlighter({ mdx, cache })` receives the MDX configuration and the run's cache policy, and returns an object with `codeToHtml(code, lang?)`, or `undefined` to leave source unhighlighted. The standard Shiki backend reuses highlighted code across builds from a cache in the OS temporary directory; `cache` is `false` for `--no-cache` runs. An ordinary provider replaces the default without initializing it. Competing providers in the same precedence group fail before either initializes. Per-language `resolveHighlighter(lang)` callbacks still run before the selected provider. + +Highlighters may also return `flush(): void | Promise` to persist buffered computation results. Core awaits it after full builds, change batches, and lazy development renders, including failed batches. Resolver composition preserves the provider's flush method. Standalone consumers must await it themselves after rendering. `PersistentCache` from `@purestack/ts-util-node` supplies buffered JSON shards, separate files for large/binary values, and inter-process merging for build-time plugins and components; pass `enabled: cache` and return `flush: () => persistentCache.flush()` from a cached highlighter. ### Search providers diff --git a/packages/ts-ssg/package.json b/packages/ts-ssg/package.json index aabaa171..175f0069 100644 --- a/packages/ts-ssg/package.json +++ b/packages/ts-ssg/package.json @@ -61,7 +61,7 @@ "html-minifier-next": "^4.19.1", "logpot": "^1.0.7", "mdast-util-to-hast": "^13.2.1", - "regor": "^1.7.5", + "regor": "^1.7.6", "rehype-raw": "^7.0.0", "rehype-stringify": "^10.0.1", "remark-gfm": "^4.0.1", diff --git a/packages/ts-ssg/src/build/build-config.ts b/packages/ts-ssg/src/build/build-config.ts index 0444b09f..85aedc50 100644 --- a/packages/ts-ssg/src/build/build-config.ts +++ b/packages/ts-ssg/src/build/build-config.ts @@ -12,11 +12,10 @@ export function resolveBuildSiteConfig( plugins: readonly PureStackPlugin[], file?: SiteConfigInput, ): SiteConfig { - const config = resolveSelectedSiteConfig( - input.siteConfig ?? {}, - plugins, - file, - ) + const config: SiteConfig = { + ...resolveSelectedSiteConfig(input.siteConfig ?? {}, plugins, file), + cache: input.options?.cache !== false, + } if (input.publish?.enabled !== true) return config return { ...config, diff --git a/packages/ts-ssg/src/build/incremental/index.ts b/packages/ts-ssg/src/build/incremental/index.ts index 7290388b..b4ebb057 100644 --- a/packages/ts-ssg/src/build/incremental/index.ts +++ b/packages/ts-ssg/src/build/incremental/index.ts @@ -121,7 +121,7 @@ async function createIncrementalRuntime( const minifyScripts = publishOptions.enabled === true const failOnAssetError = publishOptions.enabled === true const mdx = { - ...(await resolveMdxBuildOptions(config.mdx, plugins)), + ...(await resolveMdxBuildOptions(config.mdx, plugins, config.cache)), contentProcessor: resolvePluginContentProcessor(plugins), } initRenderer() @@ -260,12 +260,34 @@ class IncrementalRuntime { toBuilder(): IncrementalBuilder { const bind = - (run: (...args: A) => R) => - (...args: A): R => - runWithStyleContext(this.options.styleContext, () => run(...args)) + (run: (...args: A) => Promise) => + (...args: A): Promise => + runWithStyleContext(this.options.styleContext, async () => { + let result: R + try { + result = await run(...args) + } catch (error) { + // Persist useful work from a failed batch without replacing its original error. + await Promise.resolve() + .then(() => this.context.mdx?.highlighter?.flush?.()) + .catch(() => {}) + throw error + } + await this.context.mdx?.highlighter?.flush?.() + return result + }) + const buildAll = bind(this.buildAll) return { prepareForRequests: bind(this.prepareForRequests), - buildAll: bind(this.buildAll), + buildAll: async (reason) => { + const buildStartMs = Date.now() + const result = await buildAll(reason) + this.log.info('build completed', { + ...result, + totalDurationMs: Date.now() - buildStartMs, + }) + return result + }, applyChange: bind(this.applyChange), applyChanges: bind(this.applyChanges), renderIfDirtyByOutPath: bind(this.renderIfDirtyByOutPath), @@ -396,7 +418,6 @@ class IncrementalRuntime { buildAll = async (reason: string): Promise => { this.context.styleCacheKey = createScriptCacheKey() - const buildStartMs = Date.now() const hooks = this.resolveBuildHooks() this.scriptCacheKeys.clear() @@ -422,19 +443,13 @@ class IncrementalRuntime { }) const styleResult = await this.writeStylesWithHooks(hooks) this.preparedAssetRevision = this.pageAssetRevision - const finalResult = await this.finalizeBuild({ + return this.finalizeBuild({ ...prepared, assetFiles: [...copiedAssets.files, ...scriptAssetFiles], pages, styleResult, hooks, }) - - this.log.info('build completed', { - ...finalResult, - totalDurationMs: Date.now() - buildStartMs, - }) - return finalResult } private resolveBuildHooks(): BuildHooks { diff --git a/packages/ts-ssg/src/build/incremental/support.ts b/packages/ts-ssg/src/build/incremental/support.ts index af6dd050..fbb64a3d 100644 --- a/packages/ts-ssg/src/build/incremental/support.ts +++ b/packages/ts-ssg/src/build/incremental/support.ts @@ -126,10 +126,11 @@ export class ManifestContentIndex { export async function resolveMdxBuildOptions( mdx: Partial | undefined, plugins: readonly PureStackPlugin[] = [], + cache = true, ): Promise { const highlighter = resolvePluginHighlighter( plugins, - await createPluginHighlighter(plugins, mdx ?? {}), + await createPluginHighlighter(plugins, { mdx: mdx ?? {}, cache }), ) return { highlighter, diff --git a/packages/ts-ssg/src/build/manifest.ts b/packages/ts-ssg/src/build/manifest.ts index cd59f110..7cfb1a19 100644 --- a/packages/ts-ssg/src/build/manifest.ts +++ b/packages/ts-ssg/src/build/manifest.ts @@ -183,7 +183,9 @@ function manifestConfigEqual(left: ManifestConfig, right: ManifestConfig) { } function createManifestConfigSignature(config: SiteConfig) { - return createHash('sha256').update(JSON.stringify(config)).digest('hex') + // Caching changes build time, not output, so it never invalidates pages. + const { cache: _cache, ...output } = config + return createHash('sha256').update(JSON.stringify(output)).digest('hex') } function parseManifestJson( diff --git a/packages/ts-ssg/src/build/site.ts b/packages/ts-ssg/src/build/site.ts index 91890654..80c65c55 100644 --- a/packages/ts-ssg/src/build/site.ts +++ b/packages/ts-ssg/src/build/site.ts @@ -63,6 +63,8 @@ export interface BuildHooks { export interface BuildOptions { cleanOutDir?: boolean + /** `false` skips persistent plugin caches outside the output directory. Defaults to `true`. */ + cache?: boolean writeErrorPages?: boolean /** Include the standard component, theme, browser-runtime, highlighting, and search plugins. */ defaultPlugins?: boolean diff --git a/packages/ts-ssg/src/cli-runner.ts b/packages/ts-ssg/src/cli-runner.ts index c2292f64..691cdb3c 100644 --- a/packages/ts-ssg/src/cli-runner.ts +++ b/packages/ts-ssg/src/cli-runner.ts @@ -126,6 +126,11 @@ function parseCliArgs(args: string[]): CliState { state.input.build.options ??= {} state.input.build.options.cleanOutDir = true } + if (options.includes('--no-cache')) { + state.input.build ??= {} + state.input.build.options ??= {} + state.input.build.options.cache = false + } if (command === 'publish') { state.input.build ??= {} state.input.build.publish = { enabled: true } @@ -169,10 +174,16 @@ function assertKnownOptions( } function resolveFlagOptions(command: Exclude) { - if (command === 'build') return ['--clean'] + if (command === 'build') return ['--clean', '--no-cache'] if (command === 'serve') - return ['--clean', '--no-watch', '--no-reload', '--full-render'] - return [] + return [ + '--clean', + '--no-cache', + '--no-watch', + '--no-reload', + '--full-render', + ] + return ['--no-cache'] } function resolveValueOptions(command: Exclude) { @@ -207,9 +218,9 @@ class CliUsageError extends Error { const USAGE = `Usage: purestack --version - purestack build --content [--clean] - purestack serve --content [--host ] [--port ] [--clean] [--no-watch] [--no-reload] [--full-render] - purestack publish --content + purestack build --content [--clean] [--no-cache] + purestack serve --content [--host ] [--port ] [--clean] [--no-cache] [--no-watch] [--no-reload] [--full-render] + purestack publish --content [--no-cache] Commands: build Build a content directory into its configured outDir. @@ -217,5 +228,7 @@ Commands: publish Clean and build a publish artifact using the configured publishDir. Options: + --clean Empty the output directory and ignore its incremental manifest. + --no-cache Skip persistent plugin caches outside the output directory. --help, -h Print this usage summary. --version, -v Print the installed PureStack version.` diff --git a/packages/ts-ssg/src/config/core-config.ts b/packages/ts-ssg/src/config/core-config.ts index 13f5c809..8ca60cee 100644 --- a/packages/ts-ssg/src/config/core-config.ts +++ b/packages/ts-ssg/src/config/core-config.ts @@ -94,6 +94,7 @@ export function resolveCoreSiteConfig( sitemap, i18n, mdx, + cache: true, } return applyConfigResolvers(config, resolvers, input, fileConfig) } diff --git a/packages/ts-ssg/src/index.ts b/packages/ts-ssg/src/index.ts index 53f10c53..e638bcef 100644 --- a/packages/ts-ssg/src/index.ts +++ b/packages/ts-ssg/src/index.ts @@ -75,6 +75,7 @@ export { export { definePlugin, type GeneratedPage, + type HighlighterConfig, type PageGenerationContext, type PureStackMarkdown, type PureStackPlugin, diff --git a/packages/ts-ssg/src/plugins/config.test.ts b/packages/ts-ssg/src/plugins/config.test.ts index b8e7cd1e..913430e0 100644 --- a/packages/ts-ssg/src/plugins/config.test.ts +++ b/packages/ts-ssg/src/plugins/config.test.ts @@ -86,7 +86,8 @@ describe('plugin configuration', () => { markdown: { createHighlighter(config) { calls++ - expect(config.highlighter).toBe('custom-engine') + expect(config.mdx.highlighter).toBe('custom-engine') + expect(config.cache).toBe(true) return { codeToHtml: () => '
Custom output
', diff --git a/packages/ts-ssg/src/plugins/diagnostics.test.ts b/packages/ts-ssg/src/plugins/diagnostics.test.ts index 1292e8c2..57d54622 100644 --- a/packages/ts-ssg/src/plugins/diagnostics.test.ts +++ b/packages/ts-ssg/src/plugins/diagnostics.test.ts @@ -278,7 +278,7 @@ describe('plugin diagnostics and composition', () => { }, }, ], - {}, + { mdx: {}, cache: true }, ) await expect(backend?.codeToHtml('code')).rejects.toMatchObject({ cause, diff --git a/packages/ts-ssg/src/plugins/highlighter-flush.test.ts b/packages/ts-ssg/src/plugins/highlighter-flush.test.ts new file mode 100644 index 00000000..e1e77b44 --- /dev/null +++ b/packages/ts-ssg/src/plugins/highlighter-flush.test.ts @@ -0,0 +1,184 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import { h } from '@purestack/ts-html' +import { PersistentCache } from '@purestack/ts-util-node' +import { disableLogger, getLogger } from 'logpot' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { createIncrementalBuilder } from '../build/incremental' +import { resolveMdxBuildOptions } from '../build/incremental/support' +import { buildSite } from '../build/site' +import { makeRepoTempDir } from '../test/repoTempDir' +import type { PureStackPlugin } from './plugin' + +describe('highlighter flush lifecycle', () => { + beforeAll(() => disableLogger()) + afterAll(() => getLogger().close()) + + it('persists after full builds, lazy pages, and change batches through resolver composition', async () => { + const root = await makeRepoTempDir('.tmp-highlighter-flush-') + try { + const contentDir = path.join(root, 'content'), + outDir = path.join(root, 'out') + const cacheOptions = { + directory: root, + namespace: 'highlight', + version: '1', + flushIntervalMs: 0, + } + await fs.mkdir(contentDir) + const source = (value: string) => + `# Page\n\n\x60\x60\x60ts\n${value}\n\x60\x60\x60\n` + const first = path.join(contentDir, 'index.md') + const second = path.join(contentDir, 'two.md') + await fs.writeFile(first, source('first')) + await fs.writeFile(second, source('second')) + let flushes = 0 + const writes: number[] = [] + const plugins: PureStackPlugin[] = [ + { + name: 'buffered-highlighter', + templates: { + doc: ({ head, bodyHtml }) => + h('html').push(head, h('body').raw(bodyHtml)), + }, + markdown: { + createHighlighter({ cache: enabled }) { + const cache = new PersistentCache({ + ...cacheOptions, + enabled, + }) + return { + codeToHtml: (code) => + cache.getOrSetAsync( + code, + async () => `
${code}
`, + ), + async flush() { + await cache.flush() + flushes++ + }, + } + }, + }, + hooks: { + onPageWritten() { + writes.push(flushes) + }, + }, + }, + { name: 'resolver', markdown: { resolveHighlighter: () => null } }, + ] + const input = { + siteConfig: { rootDir: root, contentDir, outDir }, + options: { defaultPlugins: false, plugins }, + } + await buildSite(input) + expect(writes).toEqual([0, 0]) + expect(flushes).toBe(1) + expect( + await new PersistentCache(cacheOptions).get('first'), + ).toContain('first') + expect( + await new PersistentCache(cacheOptions).get('second'), + ).toContain('second') + + const builder = await createIncrementalBuilder(input) + await builder.prepareForRequests() + await fs.writeFile(first, source('lazy')) + await builder.applyChange(first) + expect(await builder.renderByUrlPath('/')).toBe(true) + expect( + await new PersistentCache(cacheOptions).get('lazy'), + ).toContain('lazy') + + await fs.writeFile(first, source('changed-one')) + await fs.writeFile(second, source('changed-two')) + await builder.applyChanges([first, second]) + await builder.renderByUrlPath('/') + await builder.renderByUrlPath('/two/') + expect( + await new PersistentCache(cacheOptions).get('changed-one'), + ).toContain('changed-one') + expect( + await new PersistentCache(cacheOptions).get('changed-two'), + ).toContain('changed-two') + } finally { + await fs.rm(root, { recursive: true, force: true }) + } + }) + + it('flushes after failure and preserves the render error if flushing also fails', async () => { + const root = await makeRepoTempDir('.tmp-highlighter-failure-') + try { + const contentDir = path.join(root, 'content') + await fs.mkdir(contentDir) + await fs.writeFile( + path.join(contentDir, 'index.md'), + '# Page\n\n```ts\nfail\n```\n', + ) + let flushes = 0 + await expect( + buildSite({ + siteConfig: { + rootDir: root, + contentDir, + outDir: path.join(root, 'out'), + }, + options: { + defaultPlugins: false, + plugins: [ + { + name: 'broken', + markdown: { + createHighlighter: () => ({ + codeToHtml() { + throw new Error('render failed') + }, + flush() { + flushes++ + throw new Error('flush failed') + }, + }), + }, + }, + ], + }, + }), + ).rejects.toThrow('render failed') + expect(flushes).toBe(1) + } finally { + await fs.rm(root, { recursive: true, force: true }) + } + }) + + it('validates flush methods and attributes failures to the highlighter provider', async () => { + await expect( + resolveMdxBuildOptions({}, [ + { + name: 'invalid', + markdown: { + createHighlighter: () => + ({ codeToHtml: () => '', flush: 42 }) as never, + }, + }, + ]), + ).rejects.toThrow('must return a code highlighter') + const options = await resolveMdxBuildOptions({}, [ + { + name: 'broken', + markdown: { + createHighlighter: () => ({ + codeToHtml: () => '', + async flush() { + throw new Error('write failed') + }, + }), + resolveHighlighter: () => null, + }, + }, + ]) + await expect(options.highlighter?.flush?.()).rejects.toThrow( + 'Plugin "broken" failed in markdown.createHighlighter.flush: write failed', + ) + }) +}) diff --git a/packages/ts-ssg/src/plugins/plugin.ts b/packages/ts-ssg/src/plugins/plugin.ts index 9decd7b3..d81160b1 100644 --- a/packages/ts-ssg/src/plugins/plugin.ts +++ b/packages/ts-ssg/src/plugins/plugin.ts @@ -3,7 +3,6 @@ import type { MdxCodeHighlighter, PageTemplateMap, SiteConfig, - SiteMdxConfig, SiteSearch, SiteStyles, } from '@purestack/ts-common' @@ -11,13 +10,14 @@ import { registerSkin, type ThemeSkin, themeSkins } from '@purestack/ts-style' import { escapeHtml, isPlainObject } from '@purestack/ts-util' import { awaitPlugin, callPlugin } from './errors' -import type { PureStackPlugin } from './types' +import type { HighlighterConfig, PureStackPlugin } from './types' export { composePluginHooks } from './hooks' export { resolvePluginContentProcessor } from './markdown' export { definePlugin, type GeneratedPage, + type HighlighterConfig, type PageGenerationContext, type PureStackMarkdown, type PureStackPlugin, @@ -135,9 +135,9 @@ export function resolvePluginStyles( /** Ordinary providers override defaults; competing providers fail before initialization. */ export async function createPluginHighlighter( plugins: readonly PureStackPlugin[], - config: Partial, + config: HighlighterConfig, ): Promise { - if (config.disableHighlighter) return undefined + if (config.mdx.disableHighlighter) return undefined const providers = plugins.filter( (plugin) => plugin.markdown?.createHighlighter, ) @@ -157,13 +157,25 @@ export async function createPluginHighlighter( const highlighter = await owner.markdown?.createHighlighter?.(config) if ( highlighter !== undefined && - typeof highlighter?.codeToHtml !== 'function' + (typeof highlighter?.codeToHtml !== 'function' || + (highlighter.flush !== undefined && + typeof highlighter.flush !== 'function')) ) throw new Error( 'markdown.createHighlighter must return a code highlighter or undefined.', ) return ( highlighter && { + ...(highlighter.flush + ? { + flush: () => + awaitPlugin( + owner.name, + 'markdown.createHighlighter.flush', + () => highlighter.flush?.(), + ), + } + : {}), codeToHtml: (code: string, lang?: string) => awaitPlugin(owner.name, 'markdown.createHighlighter.codeToHtml', () => highlighter.codeToHtml(code, lang), @@ -218,6 +230,7 @@ export function resolvePluginHighlighter( ) if (resolvers.length === 0) return fallback return { + ...(fallback?.flush ? { flush: () => fallback.flush?.() } : {}), async codeToHtml(code, lang) { const normalizedLang = lang?.trim().toLowerCase() ?? '' for (const { plugin, resolve } of resolvers) { diff --git a/packages/ts-ssg/src/plugins/types.ts b/packages/ts-ssg/src/plugins/types.ts index 34d7313e..6e374d9d 100644 --- a/packages/ts-ssg/src/plugins/types.ts +++ b/packages/ts-ssg/src/plugins/types.ts @@ -87,10 +87,16 @@ export interface GeneratedPage { source: string | (() => string | Promise) } +/** What a highlighter may depend on: Markdown settings and the run's cache policy. */ +export interface HighlighterConfig { + mdx: Partial + cache: SiteConfig['cache'] +} + export interface PureStackMarkdown { /** Creates the site's default code highlighter once per build/session. */ createHighlighter?: ( - config: Partial, + config: HighlighterConfig, ) => MdxCodeHighlighter | undefined | Promise /** Resolves a lowercase language (empty when absent); null lets the next highlighter handle it. */ resolveHighlighter?: ( diff --git a/packages/ts-util-node/README.md b/packages/ts-util-node/README.md index 789a98a5..f7682e68 100644 --- a/packages/ts-util-node/README.md +++ b/packages/ts-util-node/README.md @@ -1,8 +1,119 @@ # @purestack/ts-util-node -File path helpers for PureStack's Node.js build tools. `ensureDir(filePath)` +Caching and file path helpers for PureStack's Node.js build tools. `ensureDir(filePath)` creates the parent directory for a file, and `replaceExt(filePath, newExt)` returns the path with a different extension. Use this package in build scripts and other Node.js code. Browser-safe helpers live in `@purestack/ts-util`. + +## Persistent build cache + +`PersistentCache` reuses expensive computations across builds. Small text/JSON +values live in 32 lazily loaded JSON shards. Writes accumulate in memory; a flush +merges each dirty shard once under an inter-process lock and publishes it +atomically. Clean shards are untouched. Binary values and text over 64 KiB live +in separate files, with references in the shards. Neither is base64-encoded into +JSON. Bounded LRU caches hold decoded values and loaded shards. + +Concurrent `getOrSetAsync` calls for one key share a lookup and computation within +an instance. Separate processes may compute the same key independently; shard +merges preserve their unrelated additions and deletions. + +Create the cache once in your plugin or build-time component factory: + +```ts +import { PersistentCache } from '@purestack/ts-util-node' + +function createSnippetRenderer(config: { cache: boolean; locale: string }) { + const cache = new PersistentCache({ + namespace: 'snippet-html', + version: 'renderer-v1', + enabled: config.cache, + }) + + return { + render: (source: string) => + cache.getOrSetAsync(JSON.stringify([config.locale, source]), async () => { + return renderSnippet(source, config.locale) + }), + flush: () => cache.flush(), + } +} + +const renderer = createSnippetRenderer({ cache: true, locale: 'en' }) +try { + const html = await renderer.render('example source') + // Use html in your output. +} finally { + await renderer.flush() +} +``` + +`renderSnippet` represents your expensive rendering function. Cache only results +that can be reused without repeating side effects. Component registration, +dependency tracking, style registration, and asset emission must still run on a +cache hit. Cached values are shared by reference in memory; treat them as immutable. + +The namespace identifies the owner and accepts lowercase letters, digits, +underscores, and hyphens. **Include every output-affecting input in the key**: +source, component props, locale, configuration, and relevant dependency versions. +Change `version` when the implementation or serialized format changes. There is +no automatic source-file invalidation. Namespaces are shared across sites by +default; include site identity in keys when the result depends on the site. + +Values default to JSON serialization. For raw strings or binary data, supply a +codec (and change the cache version if changing an existing codec): + +```ts +const htmlCache = new PersistentCache({ + namespace: 'rendered-html', + version: '1', + codec: { + encode: (html) => html, + decode: (bytes) => bytes.toString('utf8'), + }, +}) +``` + +`get(key)` returns a value or `undefined`. `set(key, value)` and `delete(key)` +buffer mutations that are immediately visible to that instance. Other instances +see persisted changes after `flush()`. Await producers before flushing, and await +outstanding operations on a key before deleting or replacing it. Computation +errors propagate and can be retried. Unavailable storage and codec errors skip +persistence or become misses; they do not fail the computation. Failed writes +remain buffered for another flush, subject to the buffer limits. + +**Always await `flush()` at the end of a batch, including on failure.** A +best-effort timer flushes during long-running use every 2 seconds after writes +begin. It does not keep Node alive, and there is no process exit handler. Buffer +pressure may flush early; if storage remains unavailable, failed mutations are +discarded to keep memory bounded. `enabled: false` bypasses memory, disk, timers, +and cleanup. Passing `enabled: config.cache` honors PureStack's `--no-cache`. + +For highlighters, return an optional `flush()` alongside `codeToHtml()`. PureStack +awaits it after full builds, change batches, and lazy development renders, also +when a batch fails. Standalone highlighter users must await it themselves. +Other plugin caches can use existing lifecycle hooks or explicit `try/finally` +around their computation batches. + +| Option | Default | Purpose | +| --- | --- | --- | +| `directory` | `os.tmpdir()/purestack-cache` | Parent of the namespace directory | +| `maxMemoryEntries` | `4096` | Maximum decoded values and buffered mutations per instance | +| `maxMemoryBytes` | `16777216` (16 MiB) | Approximate budget split across decoded values, loaded shards, and buffered writes; encoding and in-flight flushes need temporary memory | +| `maxAgeDays` | `30` | Disk retention for unused entries; `0` disables cleanup | +| `flushIntervalMs` | `2000` | Best-effort periodic persistence; `0` disables the timer | + +Files live under `//00.json` through `1f.json`, with +larger values in `blobs/.cache`. The key hash covers the version and input +key. Each entry records its own last-used day: reading one entry does not retain +every other entry in its shard. Retention updates are buffered too. Housekeeping +runs at most hourly during flushing and removes idle entries, orphan blobs, and +abandoned temporary files under the same shard locks used by writers. `prune()` +flushes then requests cleanup explicitly. Retention is storage housekeeping, +not a freshness guarantee or a total disk-size limit. + +This cache stores independent values. Use specialized storage when a result +requires publishing a group of files together, as image variants do. Browser +components should use `Cache` from `@purestack/ts-util` instead. diff --git a/packages/ts-util-node/package.json b/packages/ts-util-node/package.json index 2e698015..e3de1f48 100644 --- a/packages/ts-util-node/package.json +++ b/packages/ts-util-node/package.json @@ -33,6 +33,10 @@ "README.md", "LICENSE" ], + "dependencies": { + "@purestack/ts-util": "workspace:^", + "proper-lockfile": "^4.1.2" + }, "scripts": { "build": "tsc", "lint": "biome check ./src --formatter-enabled=false --write", diff --git a/packages/ts-util-node/src/cacheStorage.ts b/packages/ts-util-node/src/cacheStorage.ts new file mode 100644 index 00000000..7a59ff50 --- /dev/null +++ b/packages/ts-util-node/src/cacheStorage.ts @@ -0,0 +1,194 @@ +import { randomUUID } from 'node:crypto' +import fs from 'node:fs/promises' +import { createRequire } from 'node:module' +import path from 'node:path' + +// proper-lockfile is CommonJS. Keep its small, private boundary out of the public declarations. +const lockfile = createRequire(import.meta.url)('proper-lockfile') as { + lock( + file: string, + options: { + realpath: boolean + retries: { retries: number; minTimeout: number; maxTimeout: number } + onCompromised(error: Error): void + }, + ): Promise<() => Promise> +} + +export const CACHE_SHARDS = 32 +export const CACHE_DAY_MS = 86_400_000 +const INLINE_LIMIT = 64 * 1024 + +/** Null data points to a separate blob; text stays directly in the JSON shard. */ +export type StoredCacheEntry = [day: number, data: string | null] +export type CacheShard = Record +export type CacheChange = + | { kind: 'put'; entry: StoredCacheEntry; blob?: Uint8Array } + | { kind: 'touch'; day: number } + | { kind: 'delete' } + +export function cacheShardIndex(id: string) { + return Number.parseInt(id.slice(0, 2), 16) % CACHE_SHARDS +} + +export function cacheShardFile(directory: string, index: number) { + return path.join(directory, `${index.toString(16).padStart(2, '0')}.json`) +} + +export function cacheBlobFile(directory: string, id: string) { + return path.join(directory, 'blobs', `${id}.cache`) +} + +export function encodeCacheChange( + data: string | Uint8Array, + day: number, +): CacheChange & { kind: 'put' } { + return typeof data === 'string' && Buffer.byteLength(data) <= INLINE_LIMIT + ? { kind: 'put', entry: [day, data] } + : { + kind: 'put', + entry: [day, null], + blob: typeof data === 'string' ? Buffer.from(data) : data, + } +} + +export async function readCacheShard(file: string): Promise { + try { + return JSON.parse(await fs.readFile(file, 'utf8')) as CacheShard + } catch { + return {} + } +} + +async function atomicWrite( + file: string, + data: string | Uint8Array, + assertLocked: () => void, +) { + const temporary = `${file}.${randomUUID()}.tmp` + try { + await fs.mkdir(path.dirname(file), { recursive: true }) + await fs.writeFile(temporary, data, { flag: 'wx' }) + assertLocked() + await fs.rename(temporary, file) + } catch (error) { + await fs.rm(temporary, { force: true }).catch(() => {}) + throw error + } +} + +/** Reload and merge under one shard lock, including its binary files and housekeeping. */ +export async function updateCacheShard( + directory: string, + index: number, + changes: ReadonlyMap, + cutoffDay?: number, + cleanFiles: readonly string[] = [], +): Promise { + const file = cacheShardFile(directory, index) + await fs.mkdir(directory, { recursive: true }) + let compromised: Error | undefined + const release = await lockfile.lock(file, { + realpath: false, + retries: { retries: 8, minTimeout: 10, maxTimeout: 100 }, + onCompromised: (error) => { + compromised = error + }, + }) + const assertLocked = () => { + if (compromised) throw compromised + } + try { + const entries = await readCacheShard(file) + let changed = false + for (const [id, change] of changes) { + if (change.kind === 'put') { + if (change.blob) + await atomicWrite( + cacheBlobFile(directory, id), + change.blob, + assertLocked, + ) + entries[id] = change.entry + changed = true + } else if (change.kind === 'delete') { + if (Object.hasOwn(entries, id)) { + delete entries[id] + changed = true + } + } else if (Object.hasOwn(entries, id) && entries[id][0] < change.day) { + entries[id][0] = change.day + changed = true + } + } + if (cutoffDay !== undefined) { + for (const [id, [day]] of Object.entries(entries)) { + if (day < cutoffDay) { + delete entries[id] + changed = true + } + } + } + assertLocked() + if (changed) await atomicWrite(file, JSON.stringify(entries), assertLocked) + for (const target of cleanFiles) { + const name = path.basename(target) + const id = name.slice(0, 64) + // Retained blob entries remain valid regardless of the blob's modification time. + if (name === `${id}.cache` && entries[id]?.[1] === null) continue + const stat = await fs.stat(target).catch(() => undefined) + if ( + stat && + cutoffDay !== undefined && + stat.mtimeMs < cutoffDay * CACHE_DAY_MS + ) { + assertLocked() + await fs.rm(target, { force: true }).catch(() => {}) + } + } + return entries + } finally { + await release().catch(() => {}) + } +} + +/** Only files owned by this layout are eligible; caller cleans them under their shard's lock. */ +export async function listCacheFiles( + directory: string, +): Promise> { + const result = new Map() + const add = (index: number, file: string) => { + const files = result.get(index) ?? [] + files.push(file) + result.set(index, files) + } + const entries = await fs + .readdir(directory, { withFileTypes: true }) + .catch(() => []) + for (const entry of entries) { + if (!entry.isFile()) continue + const match = /^([01][a-f0-9])\.json(?:\.[a-f0-9-]{36}\.tmp)?$/.exec( + entry.name, + ) + if (match) { + const index = Number.parseInt(match[1], 16) + if (!result.has(index)) result.set(index, []) + if (entry.name.endsWith('.tmp')) + add(index, path.join(directory, entry.name)) + } + } + const blobs = await fs + .readdir(path.join(directory, 'blobs'), { withFileTypes: true }) + .catch(() => []) + for (const entry of blobs) { + if ( + entry.isFile() && + /^[a-f0-9]{64}\.cache(?:\.[a-f0-9-]{36}\.tmp)?$/.test(entry.name) + ) + add( + cacheShardIndex(entry.name), + path.join(directory, 'blobs', entry.name), + ) + } + return result +} diff --git a/packages/ts-util-node/src/index.ts b/packages/ts-util-node/src/index.ts index 9a501caa..f5aae062 100644 --- a/packages/ts-util-node/src/index.ts +++ b/packages/ts-util-node/src/index.ts @@ -1 +1,6 @@ export { ensureDir, replaceExt } from './fs' +export { + PersistentCache, + type PersistentCacheCodec, + type PersistentCacheOptions, +} from './persistentCache' diff --git a/packages/ts-util-node/src/persistentCache.test.ts b/packages/ts-util-node/src/persistentCache.test.ts new file mode 100644 index 00000000..81cd1fdb --- /dev/null +++ b/packages/ts-util-node/src/persistentCache.test.ts @@ -0,0 +1,430 @@ +import { fork } from 'node:child_process' +import { createHash } from 'node:crypto' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { PersistentCache, type PersistentCacheOptions } from './index' + +describe('PersistentCache', () => { + let directory: string + beforeEach(async () => { + directory = await fs.mkdtemp( + path.join(os.tmpdir(), 'purestack-persistent-cache-'), + ) + }) + afterEach(async () => { + vi.restoreAllMocks() + await fs.rm(directory, { recursive: true, force: true }) + }) + + function cache(options: Partial> = {}) { + return new PersistentCache({ + directory, + namespace: 'test', + version: '1', + flushIntervalMs: 0, + ...options, + }) + } + async function files(root = directory): Promise { + const entries = await fs.readdir(root, { withFileTypes: true }) + return ( + await Promise.all( + entries.map((entry) => { + const file = path.join(root, entry.name) + return entry.isDirectory() ? files(file) : [file] + }), + ) + ).flat() + } + function id(key: string, version = '1') { + return createHash('sha256') + .update(JSON.stringify([version, key])) + .digest('hex') + } + + it('buffers mutations, exposes them immediately, and persists only at flush', async () => { + const instance = cache<{ count: number }>() + await instance.set('answer', { count: 42 }) + expect(await instance.get('answer')).toEqual({ count: 42 }) + expect(await fs.readdir(directory)).toEqual([]) + expect(await cache().get('answer')).toBeUndefined() + await instance.flush() + expect(await cache().get('answer')).toEqual({ count: 42 }) + expect(await cache({ namespace: 'other' }).get('answer')).toBeUndefined() + expect(await cache({ version: '2' }).get('answer')).toBeUndefined() + const otherVersion = cache({ version: '2' }) + await otherVersion.set('answer', 43) + await otherVersion.flush() + expect(await cache().get('answer')).toEqual({ count: 42 }) + }) + + it('flushes a large batch to at most 32 JSON shards and leaves a clean flush unchanged', async () => { + const instance = cache() + for (let index = 0; index < 500; index++) + await instance.set(`key-${index}`, `value-${index}`) + expect(await fs.readdir(directory)).toEqual([]) + const writes = vi.spyOn(fs, 'writeFile') + await instance.flush() + expect( + writes.mock.calls.filter(([file]) => String(file).includes('.json.')), + ).toHaveLength(32) + writes.mockClear() + await instance.flush() + expect(writes).not.toHaveBeenCalled() + const reader = cache() + for (let index = 0; index < 500; index++) + expect(await reader.get(`key-${index}`)).toBe(`value-${index}`) + }) + + it('rewrites only affected shards and merges changes with the current file', async () => { + const first = cache(), + second = cache() + for (let index = 0; index < 100; index++) + await first.set(`key-${index}`, index) + await first.flush() + await second.get('key-0') // Load a snapshot before another writer changes it. + const before = new Map( + await Promise.all( + (await files()).map( + async (file) => [file, await fs.readFile(file, 'utf8')] as const, + ), + ), + ) + const writes = vi.spyOn(fs, 'writeFile') + await first.set('key-0', 999) + await first.flush() + expect( + writes.mock.calls.filter(([file]) => String(file).includes('.json.')), + ).toHaveLength(1) + const changed = [] + for (const [file, data] of before) + if ((await fs.readFile(file, 'utf8')) !== data) changed.push(file) + expect(changed).toHaveLength(1) + await second.set('another', 1000) + await second.flush() + expect(await cache().get('key-0')).toBe(999) + expect(await cache().get('another')).toBe(1000) + }) + + it('keeps binary and large text out of JSON while preserving empty values', async () => { + const textOptions = { + codec: { + encode: (value: string) => value, + decode: (data: Buffer) => data.toString(), + }, + } + const text = cache(textOptions) + await text.set('empty', '') + await text.set('large', 'x'.repeat(100_000)) + expect(await text.get('large')).toHaveLength(100_000) + await text.flush() + expect( + await cache(textOptions).getOrSetAsync('empty', async () => 'wrong'), + ).toBe('') + expect(await cache(textOptions).get('large')).toHaveLength(100_000) + const binaryOptions = { + codec: { + encode: (value: Uint8Array) => value, + decode: (data: Buffer) => new Uint8Array(data), + }, + } + const binary = cache(binaryOptions) + await binary.set('binary', new Uint8Array([0, 255, 128])) + await binary.flush() + expect(await cache(binaryOptions).get('binary')).toEqual( + new Uint8Array([0, 255, 128]), + ) + expect( + (await files()).filter((file) => file.endsWith('.cache')), + ).toHaveLength(2) + for (const file of (await files()).filter((file) => file.endsWith('.json'))) + expect((await fs.stat(file)).size).toBeLessThan(1000) + }) + + it('coalesces same-key computations, allows independent keys, and retries loader errors', async () => { + const instance = cache() + let calls = 0 + let release!: () => void + const gate = new Promise((resolve) => { + release = resolve + }) + const pending = Array.from({ length: 20 }, () => + instance.getOrSetAsync('same', async () => { + calls++ + await gate + return 'shared' + }), + ) + try { + expect(await instance.getOrSetAsync('other', async () => 'ready')).toBe( + 'ready', + ) + } finally { + release() + } + expect(await Promise.all(pending)).toEqual(Array(20).fill('shared')) + expect(calls).toBe(1) + await expect( + instance.getOrSetAsync('retry', async () => { + throw new Error('Computation failed') + }), + ).rejects.toThrow('Computation failed') + expect(await instance.getOrSetAsync('retry', async () => 'retried')).toBe( + 'retried', + ) + await instance.flush() + expect( + await cache().getOrSetAsync('same', async () => { + throw new Error('Should not compute') + }), + ).toBe('shared') + }) + + it('merges concurrent processes writing different keys into the same shard', async () => { + const script = path.join(directory, 'worker.mjs') + await fs.writeFile( + script, + ` +import { createHash } from 'node:crypto'; +import { PersistentCache } from ${JSON.stringify(new URL('./index.ts', import.meta.url).href)}; +const cache = new PersistentCache({ directory: process.argv[2], namespace: 'test', version: '1', flushIntervalMs: 0 }); +const keys = []; +for (let i = 0; keys.length < 20; i++) { + const key = process.argv[3] + '-' + i; + const hash = createHash('sha256').update(JSON.stringify(['1', key])).digest('hex'); + if (parseInt(hash.slice(0, 2), 16) % 32 === 0) { keys.push(key); await cache.set(key, key); } +} +process.once('message', async () => { await cache.flush(); process.disconnect(); }); +process.send(keys); +`, + ) + const children = Array.from({ length: 4 }, (_, index) => + fork(script, [directory, `worker-${index}`], { + execArgv: ['--import', 'tsx'], + silent: true, + }), + ) + const exits = children.map( + (child) => + new Promise((resolve, reject) => { + let stderr = '' + child.stderr?.on('data', (chunk) => { + stderr += chunk + }) + child.once('error', reject) + child.once('exit', (code) => + code === 0 + ? resolve() + : reject(new Error(stderr || `Worker exited ${code}`)), + ) + }), + ) + try { + const keys = await Promise.all( + children.map((child, index) => + Promise.race([ + new Promise((resolve) => + child.once('message', (message) => resolve(message as string[])), + ), + exits[index].then(() => { + throw new Error('Worker exited before becoming ready') + }), + ]), + ), + ) + for (const child of children) child.send('flush') + await Promise.all(exits) + const reader = cache() + for (const key of keys.flat()) expect(await reader.get(key)).toBe(key) + } finally { + for (const child of children) if (child.exitCode === null) child.kill() + } + }, 15000) + + it('bypasses memory, files, timers, and cleanup when disabled', async () => { + const instance = cache({ enabled: false, flushIntervalMs: 1 }) + let calls = 0 + await instance.set('key', 99) + expect(await instance.get('key')).toBeUndefined() + expect(await instance.getOrSetAsync('key', async () => ++calls)).toBe(1) + expect(await instance.getOrSetAsync('key', async () => ++calls)).toBe(2) + await instance.delete('key') + await instance.flush() + await instance.prune() + expect(await fs.readdir(directory)).toEqual([]) + }) + + it('bounds pending writes under storage failures and retries when storage becomes available', async () => { + const blocked = path.join(directory, 'blocked') + await fs.writeFile(blocked, '') + const instance = cache({ directory: blocked, maxMemoryEntries: 2 }) + for (let i = 0; i < 10; i++) { + await instance.set(`key-${i}`, i) + expect(await instance.get(`key-${i}`)).toBe(i) + } + await instance.flush() + await fs.rm(blocked) + await instance.flush() + expect(await cache({ directory: blocked }).get('key-9')).toBe(9) + }) + + it('flushes early under memory pressure without dropping concurrent producers', async () => { + const instance = cache({ + maxMemoryEntries: 2, + maxMemoryBytes: 2048, + }) + await Promise.all( + Array.from({ length: 30 }, (_, index) => + instance.set(`key-${index}`, index), + ), + ) + await instance.flush() + const reader = cache() + for (let i = 0; i < 30; i++) expect(await reader.get(`key-${i}`)).toBe(i) + }) + + it('keeps mutations made during a flush readable and persists them in the next batch', async () => { + const instance = cache({ maxMemoryEntries: 1 }) + await instance.set('one', 'first') + let release!: () => void, entered!: () => void + const gate = new Promise((resolve) => { + release = resolve + }) + const started = new Promise((resolve) => { + entered = resolve + }) + const original = fs.writeFile + let paused = false + vi.spyOn(fs, 'writeFile').mockImplementation(async (...args) => { + if (!paused && String(args[0]).includes('.json.')) { + paused = true + entered() + await gate + } + return original(...args) + }) + const flushing = instance.flush() + await started + try { + await instance.set('two', 'second') + expect(await instance.get('one')).toBe('first') + expect(await instance.get('two')).toBe('second') + } finally { + release() + await flushing + } + await instance.flush() + expect(await cache().get('one')).toBe('first') + expect(await cache().get('two')).toBe('second') + }) + + it('persists deletions without another writer resurrecting a stale snapshot', async () => { + const first = cache(), + second = cache() + await first.set('remove', 1) + await first.flush() + expect(await second.get('remove')).toBe(1) + await first.delete('remove') + expect(await first.get('remove')).toBeUndefined() + await first.flush() + await second.set('keep', 2) + await second.flush() + expect(await cache().get('remove')).toBeUndefined() + expect(await cache().get('keep')).toBe(2) + }) + + it('flushes periodically during long-running use', async () => { + const instance = cache({ flushIntervalMs: 10 }) + await instance.set('key', 42) + await vi.waitFor(async () => expect(await cache().get('key')).toBe(42)) + await instance.flush() + }) + + it('prunes idle entries and orphan blobs while retaining active blobs and foreign files', async () => { + const realNow = Date.now(), + dayMs = 86_400_000 + const now = vi.spyOn(Date, 'now').mockReturnValue(realNow - 40 * dayMs) + const old = cache({ + version: 'old', + maxAgeDays: 0, + codec: { encode: (value) => value, decode: (data) => data.toString() }, + }) + await old.set('old', 'x'.repeat(100_000)) + await old.flush() + now.mockReturnValue(realNow) + const blob = path.join( + directory, + 'test', + 'blobs', + `${id('old', 'old')}.cache`, + ) + const abandoned = `${blob}.00000000-0000-0000-0000-000000000000.tmp` + const foreign = path.join(directory, 'test', 'keep.txt') + await fs.writeFile(abandoned, '') + await fs.writeFile(foreign, '') + const past = new Date(realNow - 40 * dayMs) + for (const file of [blob, abandoned, foreign]) + await fs.utimes(file, past, past) + const active = cache({ + codec: { encode: (value) => value, decode: (data) => data.toString() }, + }) + await active.set('active', 'a'.repeat(100_000)) + await active.flush() + const activeBlob = path.join( + directory, + 'test', + 'blobs', + `${id('active')}.cache`, + ) + await fs.utimes(activeBlob, past, past) + await cache().prune() + expect(await cache({ version: 'old' }).get('old')).toBeUndefined() + expect(await files()).not.toContain(blob) + expect(await files()).not.toContain(abandoned) + expect(await fs.readFile(foreign, 'utf8')).toBe('') + expect( + await cache({ + codec: { encode: (value) => value, decode: (data) => data.toString() }, + }).get('active'), + ).toHaveLength(100_000) + }) + + it('renews individual entries rather than retaining all entries in a used shard', async () => { + const realNow = Date.now(), + dayMs = 86_400_000 + const now = vi.spyOn(Date, 'now').mockReturnValue(realNow - 40 * dayMs) + const original = cache({ maxAgeDays: 0 }) + const usedShard = Number.parseInt(id('used').slice(0, 2), 16) % 32 + const idle = Array.from( + { length: 512 }, + (_, index) => `idle-${index}`, + ).find((key) => Number.parseInt(id(key).slice(0, 2), 16) % 32 === usedShard) + if (!idle) throw new Error('Expected two keys in the same shard') + await original.set('used', 1) + await original.set(idle, 2) + await original.flush() + now.mockReturnValue(realNow) + const reader = cache() + expect(await reader.get('used')).toBe(1) + await reader.prune() + expect(await cache().get('used')).toBe(1) + expect(await cache().get(idle)).toBeUndefined() + expect(await cache({ maxAgeDays: 0 }).get('used')).toBe(1) + }) + + it('rejects invalid storage identities, limits, and flush intervals', () => { + for (const namespace of ['', '..', '../other', 'UPPER', 'with/slash']) + expect(() => cache({ namespace })).toThrow('namespace') + expect(() => cache({ version: '' })).toThrow('version') + for (const maxAgeDays of [-1, NaN, Infinity]) + expect(() => cache({ maxAgeDays })).toThrow('maxAgeDays') + for (const maxMemoryEntries of [0, 1.5, Infinity]) + expect(() => cache({ maxMemoryEntries })).toThrow('memory limits') + for (const maxMemoryBytes of [0, NaN, Infinity]) + expect(() => cache({ maxMemoryBytes })).toThrow('memory limits') + for (const flushIntervalMs of [-1, NaN, Infinity]) + expect(() => cache({ flushIntervalMs })).toThrow('flushIntervalMs') + }) +}) diff --git a/packages/ts-util-node/src/persistentCache.ts b/packages/ts-util-node/src/persistentCache.ts new file mode 100644 index 00000000..9bf14914 --- /dev/null +++ b/packages/ts-util-node/src/persistentCache.ts @@ -0,0 +1,415 @@ +import { createHash } from 'node:crypto' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { Cache } from '@purestack/ts-util' +import { + CACHE_DAY_MS, + CACHE_SHARDS, + type CacheChange, + type CacheShard, + cacheBlobFile, + cacheShardFile, + cacheShardIndex, + encodeCacheChange, + listCacheFiles, + readCacheShard, + updateCacheShard, +} from './cacheStorage' + +export interface PersistentCacheCodec { + encode(value: T): string | Uint8Array + decode(data: Buffer): T +} + +export interface PersistentCacheOptions { + /** Owner name using lowercase letters, digits, underscores, and hyphens. */ + namespace: string + /** Change whenever the computation or serialized format changes. */ + version: string + /** Parent directory. Defaults to `os.tmpdir()/purestack-cache`. */ + directory?: string + /** `false` bypasses both memory and disk. Defaults to `true`. */ + enabled?: boolean + /** Idle retention in days. Defaults to 30; 0 disables cleanup. */ + maxAgeDays?: number + /** Defaults to 4096. Also limits buffered mutations. */ + maxMemoryEntries?: number + /** Approximate combined budget for values, loaded shards, and pending writes. Defaults to 16 MiB. */ + maxMemoryBytes?: number + /** Best-effort background flush interval. Defaults to 2000 ms; 0 disables the timer. Always await flush before exit. */ + flushIntervalMs?: number + /** Defaults to JSON. Binary and text over 64 KiB are stored separately from JSON shards. */ + codec?: PersistentCacheCodec +} + +interface MemoryEntry { + value: T + bytes: number + touchedDay: number +} +interface LoadedShard { + entries: CacheShard + bytes: number + generation: number +} +type Changes = Map> + +/** Buffered build-time value cache. Await flush() after a batch to persist its mutations. */ +export class PersistentCache { + private readonly directory: string + private readonly enabled: boolean + private readonly version: string + private readonly maxAgeDays: number + private readonly maxEntries: number + private readonly bufferLimit: number + private readonly flushIntervalMs: number + private readonly memory: Cache> + private readonly shards: Cache + private readonly generations = Array(CACHE_SHARDS).fill(0) + private readonly codec: PersistentCacheCodec + private pending: Changes = new Map() + private flushing: Changes = new Map() + private pendingBytes = 0 + private pendingCount = 0 + private work: Promise = Promise.resolve() + private readonly failed = new WeakSet() + private timer?: NodeJS.Timeout + private nextMaintenance = 0 + + constructor(options: PersistentCacheOptions) { + if (!/^[a-z0-9][a-z0-9_-]*$/.test(options.namespace)) + throw new Error( + 'Cache namespace must use lowercase letters, digits, underscores, or hyphens.', + ) + if (typeof options.version !== 'string' || !options.version.length) + throw new Error('Cache version must be a non-empty string.') + this.maxAgeDays = options.maxAgeDays ?? 30 + if (!Number.isFinite(this.maxAgeDays) || this.maxAgeDays < 0) + throw new Error('Cache maxAgeDays must be a non-negative finite number.') + this.maxEntries = options.maxMemoryEntries ?? 4096 + const budget = options.maxMemoryBytes ?? 16 * 1024 * 1024 + if ( + !Number.isInteger(this.maxEntries) || + this.maxEntries < 1 || + !Number.isFinite(budget) || + budget < 1 + ) + throw new Error( + 'Cache memory limits must be positive; maxMemoryEntries must be an integer.', + ) + this.flushIntervalMs = options.flushIntervalMs ?? 2000 + if (!Number.isFinite(this.flushIntervalMs) || this.flushIntervalMs < 0) + throw new Error( + 'Cache flushIntervalMs must be a non-negative finite number.', + ) + this.directory = path.resolve( + options.directory ?? path.join(os.tmpdir(), 'purestack-cache'), + options.namespace, + ) + this.enabled = options.enabled !== false + this.version = options.version + this.bufferLimit = Math.max(1, Math.floor(budget / 3)) + this.memory = new Cache({ + maxEntries: this.maxEntries, + maxWeight: this.bufferLimit, + sizeCalculator: (entry) => entry.bytes, + }) + this.shards = new Cache({ + maxEntries: CACHE_SHARDS, + maxWeight: this.bufferLimit, + sizeCalculator: (shard) => shard.bytes, + }) + this.codec = options.codec ?? { + encode(value) { + const data = JSON.stringify(value) + if (data === undefined) + throw new Error('Value cannot be serialized as JSON.') + return data + }, + decode: (data) => JSON.parse(data.toString('utf8')) as T, + } + } + + async get(key: string): Promise { + if (!this.enabled) return undefined + const id = this.id(key) + let entry = this.memory.get(id) + if (!entry) { + entry = await this.read(id) + if (entry) this.memory.set(id, entry) + } + if (entry) await this.touch(id, entry) + return entry?.value + } + + /** Buffer a value. Large batches may flush early to keep the write buffer bounded. */ + async set(key: string, value: T): Promise { + if (!this.enabled) return + const id = this.id(key) + this.memory.set(id, await this.write(id, value)) + } + + /** Same-key lookups and computations are coalesced within this instance. */ + async getOrSetAsync(key: string, loader: () => Promise): Promise { + if (!this.enabled) return loader() + const id = this.id(key) + const entry = await this.memory.getOrSetAsync( + id, + async () => (await this.read(id)) ?? this.write(id, await loader()), + ) + await this.touch(id, entry) + return entry.value + } + + /** Buffer deletion of a settled entry. Await outstanding operations on this key first. */ + async delete(key: string): Promise { + if (!this.enabled) return + const id = this.id(key) + this.memory.delete(id) + await this.buffer(id, { kind: 'delete' }) + } + + /** Persist buffered changes; storage failures retain them for the next attempt. Await producers first. */ + flush(): Promise { + if (!this.enabled) return Promise.resolve() + clearTimeout(this.timer) + this.timer = undefined + this.work = this.work.then(async () => { + await this.flushPending() + await this.maintain() + }) + return this.work + } + + /** Flush and clean idle entries across all versions of this namespace. */ + prune(): Promise { + if (!this.enabled) return Promise.resolve() + clearTimeout(this.timer) + this.timer = undefined + this.work = this.work.then(async () => { + await this.flushPending() + if (this.maxAgeDays > 0) await this.pruneShards() + }) + return this.work + } + + private id(key: string) { + return createHash('sha256') + .update(JSON.stringify([this.version, key])) + .digest('hex') + } + + private today() { + return Math.floor(Date.now() / CACHE_DAY_MS) + } + + private async load(index: number): Promise { + const generation = this.generations[index] + const loaded = await this.shards.getOrSetAsync(index, async () => { + const entries = await readCacheShard( + cacheShardFile(this.directory, index), + ) + return { + entries, + generation, + bytes: Buffer.byteLength(JSON.stringify(entries)), + } + }) + if (loaded.generation === this.generations[index]) return loaded.entries + this.shards.delete(index) + return this.load(index) + } + + private remember(index: number, entries: CacheShard) { + this.generations[index]++ + this.shards.set(index, { + entries, + generation: this.generations[index], + bytes: Buffer.byteLength(JSON.stringify(entries)), + }) + } + + private async read(id: string): Promise | undefined> { + try { + const index = cacheShardIndex(id) + const pending = this.pending.get(index)?.get(id) + const change = + pending?.kind === 'touch' + ? (this.flushing.get(index)?.get(id) ?? pending) + : (pending ?? this.flushing.get(index)?.get(id)) + if (change?.kind === 'delete') return undefined + const stored = + change?.kind === 'put' ? change.entry : (await this.load(index))[id] + if (!stored) return undefined + const data = + stored[1] === null + ? ((change?.kind === 'put' ? change.blob : undefined) ?? + (await fs.readFile(cacheBlobFile(this.directory, id)))) + : Buffer.from(stored[1]) + return { + value: this.codec.decode(Buffer.from(data)), + bytes: data.byteLength, + touchedDay: stored[0], + } + } catch { + return undefined + } + } + + private async write(id: string, value: T): Promise> { + const entry = { + value, + bytes: Number.MAX_SAFE_INTEGER, + touchedDay: this.today(), + } + try { + const encoded = this.codec.encode(value) + entry.bytes = + typeof encoded === 'string' + ? Buffer.byteLength(encoded) + : encoded.byteLength + await this.buffer(id, encodeCacheChange(encoded, entry.touchedDay)) + } catch { + // Codec/storage failures never change the result of a successful computation. + } + return entry + } + + private async touch(id: string, entry: MemoryEntry) { + const day = this.today() + if (this.maxAgeDays === 0 || entry.touchedDay === day) return + entry.touchedDay = day + await this.buffer(id, { kind: 'touch', day }) + } + + private changeBytes(change: CacheChange) { + return ( + 128 + + (change.kind === 'put' + ? (change.blob?.byteLength ?? Buffer.byteLength(change.entry[1] ?? '')) + : 0) + ) + } + + private async buffer(id: string, change: CacheChange) { + const size = this.changeBytes(change) + while ( + this.pendingCount > 0 && + (this.pendingBytes + size > this.bufferLimit || + this.pendingCount >= this.maxEntries) + ) { + await this.flush() + // Failed persistence must not make the buffer unbounded or retry forever. + while ( + this.pendingCount > 0 && + (this.pendingBytes + size > this.bufferLimit || + this.pendingCount >= this.maxEntries) + ) { + if (!this.discardFailedChange()) break + } + } + const index = cacheShardIndex(id) + const shard = this.pending.get(index) ?? new Map() + const previous = shard.get(id) + if (change.kind === 'touch' && previous) { + if (previous.kind === 'put') previous.entry[0] = change.day + else if (previous.kind === 'touch') previous.day = change.day + return + } + if (previous) this.pendingBytes -= this.changeBytes(previous) + else this.pendingCount++ + shard.set(id, change) + this.pending.set(index, shard) + this.pendingBytes += size + if (this.flushIntervalMs > 0 && !this.timer) { + this.timer = setTimeout(() => { + this.timer = undefined + void this.flush() + }, this.flushIntervalMs) + this.timer.unref() + } + if (size > this.bufferLimit) await this.flush() + } + + private discardFailedChange() { + for (const [index, changes] of this.pending) { + for (const [id, change] of changes) { + if (!this.failed.has(change)) continue + changes.delete(id) + this.pendingBytes -= this.changeBytes(change) + this.pendingCount-- + if (!changes.size) this.pending.delete(index) + return true + } + } + return false + } + + private async flushPending() { + const batch = this.pending + this.pending = new Map() + this.pendingBytes = 0 + this.pendingCount = 0 + this.flushing = batch + try { + for (const [index, changes] of batch) { + try { + this.remember( + index, + await updateCacheShard(this.directory, index, changes), + ) + } catch { + // Keep failed changes unless a newer mutation superseded them. + const next = this.pending.get(index) ?? new Map() + for (const [id, change] of changes) { + const newer = next.get(id) + if (newer) { + if (newer.kind !== 'touch' || change.kind !== 'put') continue + change.entry[0] = Math.max(change.entry[0], newer.day) + this.pendingBytes -= this.changeBytes(newer) + this.pendingCount-- + } + next.set(id, change) + this.failed.add(change) + this.pendingBytes += this.changeBytes(change) + this.pendingCount++ + } + this.pending.set(index, next) + } + } + } finally { + this.flushing = new Map() + } + } + + private async maintain() { + if (this.maxAgeDays === 0 || Date.now() < this.nextMaintenance) return + this.nextMaintenance = Date.now() + 3_600_000 + const stat = await fs + .stat(path.join(this.directory, '.pruned')) + .catch(() => undefined) + if (!stat || stat.mtimeMs < Date.now() - 3_600_000) await this.pruneShards() + } + + private async pruneShards() { + const cutoff = this.today() - this.maxAgeDays + for (const [index, files] of await listCacheFiles(this.directory)) { + try { + this.remember( + index, + await updateCacheShard( + this.directory, + index, + new Map(), + cutoff, + files, + ), + ) + } catch { + /* Cache housekeeping is best effort. */ + } + } + await fs.writeFile(path.join(this.directory, '.pruned'), '').catch(() => {}) + } +} diff --git a/yarn.lock b/yarn.lock index 9b1cc014..c9f9d92a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1018,7 +1018,7 @@ __metadata: "@purestack/ts-style": "workspace:^" "@purestack/ts-svg-icons": "workspace:^" "@purestack/ts-util": "workspace:^" - regor: "npm:^1.7.5" + regor: "npm:^1.7.6" languageName: unknown linkType: soft @@ -1036,6 +1036,7 @@ __metadata: resolution: "@purestack/ts-highlight@workspace:packages/ts-highlight" dependencies: "@purestack/ts-common": "workspace:^" + "@purestack/ts-util-node": "workspace:^" highlight.js: "npm:^11.11.1" shiki: "npm:^3.22.0" languageName: unknown @@ -1096,7 +1097,7 @@ __metadata: "@purestack/ts-common": "workspace:^" "@purestack/ts-minidom": "workspace:^" "@purestack/ts-util": "workspace:^" - regor: "npm:^1.7.5" + regor: "npm:^1.7.6" languageName: unknown linkType: soft @@ -1136,7 +1137,7 @@ __metadata: html-minifier-next: "npm:^4.19.1" logpot: "npm:^1.0.7" mdast-util-to-hast: "npm:^13.2.1" - regor: "npm:^1.7.5" + regor: "npm:^1.7.6" rehype-raw: "npm:^7.0.0" rehype-stringify: "npm:^10.0.1" remark-gfm: "npm:^4.0.1" @@ -1188,6 +1189,9 @@ __metadata: "@purestack/ts-util-node@workspace:^, @purestack/ts-util-node@workspace:packages/ts-util-node": version: 0.0.0-use.local resolution: "@purestack/ts-util-node@workspace:packages/ts-util-node" + dependencies: + "@purestack/ts-util": "workspace:^" + proper-lockfile: "npm:^4.1.2" languageName: unknown linkType: soft @@ -4717,10 +4721,10 @@ __metadata: languageName: node linkType: hard -"regor@npm:^1.7.5": - version: 1.7.5 - resolution: "regor@npm:1.7.5" - checksum: 10c0/a6a3ebde793e1b9d4d5c168540618b1bba427e31334f1dac54dc0f93fe3940fd2db9dd47151b986bb8aed959bf95ce01ee74c0a30aca292a02aa4402e9cb998b +"regor@npm:^1.7.6": + version: 1.7.6 + resolution: "regor@npm:1.7.6" + checksum: 10c0/3427b77e524ef2ee7957c65e25958c2aa0d88709d419a0e21ddd82181798af89cfa25a73ccb6c2e1336f80d44867716073c027a0e2e9ce865dda9003a732a6cb languageName: node linkType: hard From 531087d23d13a7ae481c5217ed2e20c615b9b4bb Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Mon, 19 Oct 2026 02:33:59 +0200 Subject: [PATCH 2/2] manual npm package publisher script. --- CONTRIBUTING.md | 19 +++++++ package.json | 1 + scripts/publish-package.ts | 111 +++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 scripts/publish-package.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b615e1d..e08d5316 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,6 +53,25 @@ enforce branch protection. See GitHub's [ruleset documentation](https://docs.git Dependabot opens weekly pull requests for GitHub Actions and dependencies. Review them through the same CI and pull request process as other changes. +To publish one package manually, log in to npm once and run the command from the +repository root: + +```sh +npm login --registry=https://registry.npmjs.org +yarn publish-package ts-images +``` + +Both short names (`ts-images`) and full names (`@purestack/ts-images`, `purestack`) +are accepted. The command rebuilds and verifies all distributable bundles, packs +only the selected workspace with Yarn to resolve its `workspace:` dependencies, +then publishes its current version publicly to npm with the `latest` tag. npm's +authentication prompts remain interactive. It does not bump versions or publish +dependencies; the package's dependency versions must already be available on npm. + +Use `yarn publish-package ts-images --dry-run` to build, pack, and inspect npm's +publish preview without uploading anything. An existing name/version cannot be +published again; update the version through the normal release process first. + The existing **Publish new npm packages** workflow remains a manual operation. It only publishes packages missing from npm; it is not a version-release workflow. Use its default dry run first and review the registry results before publishing. diff --git a/package.json b/package.json index d9b35c63..3c7c783a 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "bump-version": "yarn tsx --no-warnings scripts/bump-version.ts", "bundle": "yarn tsx --no-warnings scripts/bundle.ts", "cleanup": "yarn tsx --no-warnings scripts/cleanup.ts", + "publish-package": "yarn tsx --no-warnings scripts/publish-package.ts", "publish-new-packages": "yarn tsx --no-warnings scripts/publish-new-packages.ts", "test": "vitest", "test:ci": "vitest run", diff --git a/scripts/publish-package.ts b/scripts/publish-package.ts new file mode 100644 index 00000000..06d90baf --- /dev/null +++ b/scripts/publish-package.ts @@ -0,0 +1,111 @@ +import { spawn } from 'node:child_process' +import { globSync, readFileSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const projectRoot = fileURLToPath(new URL('..', import.meta.url)) + +interface PackageJson { + name: string + version?: string + private?: boolean + exports?: Record +} + +function run(command: string, args: string[]) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: projectRoot, + stdio: 'inherit', + }) + child.once('error', reject) + child.once('exit', (code, signal) => { + if (code === 0) resolve() + else reject(new Error(`${command} failed (${signal ?? `exit ${code}`}).`)) + }) + }) +} + +async function main() { + const args = process.argv.slice(2) + if (args.includes('--help') || args.includes('-h')) { + console.log(`Build, pack, and publish one PureStack package publicly with the latest tag. + +Usage: + yarn publish-package ts-images + yarn publish-package @purestack/ts-images --dry-run + +Log in once with npm login before publishing. --dry-run builds and packs the +package, then runs npm publish --dry-run without uploading it. +`) + return + } + + const dryRun = args.includes('--dry-run') + const names = args.filter((arg) => arg !== '--dry-run') + if (names.length !== 1 || names[0].startsWith('-')) + throw new Error('Usage: yarn publish-package [--dry-run]') + + const requested = names[0] + const packageNames = new Set([requested, `@purestack/${requested}`]) + const packages = globSync('packages/*/package.json', { + cwd: projectRoot, + }).map((file) => ({ + directory: path.dirname(path.join(projectRoot, file)), + manifest: JSON.parse( + readFileSync(path.join(projectRoot, file), 'utf8'), + ) as PackageJson, + })) + const selected = packages.find((pkg) => packageNames.has(pkg.manifest.name)) + if (!selected) throw new Error(`Unknown workspace package: ${requested}`) + const { directory, manifest } = selected + if (manifest.private || !manifest.exports || !manifest.version) + throw new Error(`${manifest.name} is not a versioned, publishable package.`) + + // Yarn's npm_execpath can point to a shell shim. Use the Corepack entry directly. + const corepackRoot = process.env.COREPACK_ROOT + if (!corepackRoot) + throw new Error('Enable Corepack, then run yarn publish-package.') + const yarnCli = path.join(corepackRoot, 'dist', 'yarn.js') + const yarn = (commandArgs: string[]) => + run(process.execPath, [yarnCli, ...commandArgs]) + const tarball = path.join(directory, 'package.tgz') + console.log( + `${dryRun ? 'Preparing dry run for' : 'Preparing'} ${manifest.name}@${manifest.version}`, + ) + + // Reuse the repository's bundling and verification before packing fresh output. + await yarn(['bundle']) + await yarn(['workspace', manifest.name, 'pack', '--out', tarball]) + + const publishArgs = [ + 'publish', + tarball, + '--access', + 'public', + '--tag', + 'latest', + '--registry', + 'https://registry.npmjs.org', + ] + if (dryRun) publishArgs.push('--dry-run') + + // Inherit the terminal so npm's browser authentication and 2FA stay interactive. + if (process.platform === 'win32') { + const npmCli = path.join( + path.dirname(process.execPath), + 'node_modules', + 'npm', + 'bin', + 'npm-cli.js', + ) + await run(process.execPath, [npmCli, ...publishArgs]) + } else { + await run('npm', publishArgs) + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error) + process.exitCode = 1 +})