diff --git a/packages/ts-ssg/src/mdx/codeLanguages.test.ts b/packages/ts-ssg/src/mdx/codeLanguages.test.ts new file mode 100644 index 00000000..de6eec75 --- /dev/null +++ b/packages/ts-ssg/src/mdx/codeLanguages.test.ts @@ -0,0 +1,57 @@ +import hljs from 'highlight.js' +import { describe, expect, it, vi } from 'vitest' +import { LANGUAGE_BY_EXTENSION } from './codeLanguages' +import { createMdxHighlighter, DEFAULT_MDX_CODE_LANGS } from './highlight' +import { createHljsHighlighter } from './highlightjs' + +describe('MDX code languages', () => { + it('renders every inferred language with the default Shiki grammars', async () => { + const highlighter = await createMdxHighlighter() + const languages = new Set([ + ...DEFAULT_MDX_CODE_LANGS, + ...Object.values(LANGUAGE_BY_EXTENSION), + ]) + for (const language of languages) { + expect(await highlighter.codeToHtml('example', language)).toContain( + ' { + const engine = hljs.newInstance() + for (const language of hljs.listLanguages()) { + const grammar = hljs.getLanguage(language) + if (grammar) engine.registerLanguage(language, () => grammar) + } + const highlight = vi.spyOn(engine, 'highlight') + const highlighter = createHljsHighlighter(engine) + for (const language of DEFAULT_MDX_CODE_LANGS) { + highlight.mockClear() + highlighter.codeToHtml('example', language) + expect(highlight, language).toHaveBeenCalledOnce() + } + }) + + it.each([ + [' PY3 ', 'python', 'print(42)'], + ['cs', 'csharp', 'public class Example {}'], + ['c++', 'cpp', 'int main() {}'], + ['pwsh', 'powershell', 'Write-Host "Hello"'], + ['yml', 'yaml', 'answer: 42'], + ['shellscript', 'bash', 'echo "Hello"'], + ['objectivec', 'objective-c', '@interface Example'], + ['dos', 'bat', 'echo Hello'], + ['gql', 'graphql', 'query { user { id } }'], + ])('resolves %s with both highlighters', async (alias, language, code) => { + // Loading by the alias must also work when defaults are overridden. + const shiki = await createMdxHighlighter(undefined, [alias]) + expect(await shiki.codeToHtml(code, alias)).toBe( + await shiki.codeToHtml(code, language), + ) + const highlighter = createHljsHighlighter() + const html = highlighter.codeToHtml(code, alias) + expect(html).toBe(highlighter.codeToHtml(code, language)) + expect(html).toContain(' = { + bash: { + aliases: ['sh', 'shell', 'shellscript', 'zsh'], + extensions: ['.sh', '.bash', '.zsh'], + }, + bat: { aliases: ['batch', 'cmd', 'dos'], extensions: ['.bat', '.cmd'] }, + c: { extensions: ['.c', '.h'] }, + cmake: { extensions: ['.cmake'] }, + cpp: { + aliases: ['c++', 'cc', 'cxx', 'h++', 'hpp', 'hxx'], + extensions: ['.cc', '.cpp', '.cxx', '.hh', '.hpp', '.hxx'], + }, + csharp: { aliases: ['c#', 'cs'], extensions: ['.cs', '.csx'] }, + css: { extensions: ['.css'] }, + dart: { extensions: ['.dart'] }, + diff: { aliases: ['patch'], extensions: ['.diff', '.patch'] }, + dockerfile: { aliases: ['docker'], extensions: ['.dockerfile'] }, + elixir: { aliases: ['ex', 'exs'], extensions: ['.ex', '.exs'] }, + erlang: { aliases: ['erl'], extensions: ['.erl', '.hrl'] }, + fsharp: { aliases: ['f#', 'fs'], extensions: ['.fs', '.fsi', '.fsx'] }, + go: { aliases: ['golang'], extensions: ['.go'] }, + graphql: { aliases: ['gql'], extensions: ['.graphql', '.gql'] }, + groovy: { extensions: ['.groovy', '.gradle'] }, + html: { aliases: ['htm'], extensions: ['.html', '.htm'] }, + http: { aliases: ['https'], extensions: ['.http', '.rest'] }, + ini: { aliases: ['properties'], extensions: ['.ini', '.cfg', '.properties'] }, + java: { extensions: ['.java'] }, + javascript: { + aliases: ['js', 'cjs', 'mjs'], + extensions: ['.js', '.cjs', '.mjs'], + }, + json: { extensions: ['.json'] }, + jsonc: { extensions: ['.jsonc'] }, + jsx: { extensions: ['.jsx'] }, + kotlin: { aliases: ['kt', 'kts'], extensions: ['.kt', '.kts'] }, + less: { extensions: ['.less'] }, + lua: { extensions: ['.lua'] }, + makefile: { aliases: ['make', 'mk'], extensions: ['.mk', '.mak'] }, + markdown: { + aliases: ['md', 'mkd', 'mdown'], + extensions: ['.md', '.markdown', '.mkd', '.mdown'], + }, + mdx: { aliases: ['rmdx'], extensions: ['.mdx', '.rmdx'] }, + 'objective-c': { aliases: ['objc', 'objectivec'], extensions: ['.m'] }, + perl: { aliases: ['pl', 'pm'], extensions: ['.pl', '.pm'] }, + php: { extensions: ['.php', '.phtml'] }, + powershell: { + aliases: ['ps', 'ps1', 'pwsh'], + extensions: ['.ps1', '.psm1', '.psd1'], + }, + python: { + aliases: ['py', 'py3', 'python3'], + extensions: ['.py', '.pyw', '.pyi'], + }, + r: { extensions: ['.r'] }, + ruby: { + aliases: ['rb', 'gemspec'], + extensions: ['.rb', '.rake', '.gemspec'], + }, + rust: { aliases: ['rs'], extensions: ['.rs'] }, + scala: { extensions: ['.scala', '.sc'] }, + scss: { extensions: ['.scss'] }, + sql: { extensions: ['.sql'] }, + swift: { extensions: ['.swift'] }, + text: { aliases: ['txt', 'plaintext'], extensions: ['.txt', '.text'] }, + toml: { extensions: ['.toml'] }, + 'ts-tags': { aliases: ['lit'] }, + tsx: { extensions: ['.tsx'] }, + typescript: { + aliases: ['ts', 'cts', 'mts'], + extensions: ['.ts', '.cts', '.mts'], + }, + xml: { + aliases: ['svg', 'xhtml'], + extensions: ['.xml', '.svg', '.xhtml', '.xsd', '.xsl', '.xslt'], + }, + yaml: { aliases: ['yml'], extensions: ['.yaml', '.yml'] }, +} + +export const DEFAULT_CODE_LANGUAGES = Object.keys(codeLanguages) + +const languageAliases = new Map( + Object.entries(codeLanguages).flatMap(([language, { aliases = [] }]) => + aliases.map((alias) => [alias, language] as const), + ), +) + +export const LANGUAGE_BY_EXTENSION: Record = Object.fromEntries( + Object.entries(codeLanguages).flatMap(([language, { extensions = [] }]) => + extensions.map((extension) => [extension, language]), + ), +) + +export function normalizeCodeLanguage(value: string): string { + const normalized = value.trim().toLowerCase() + return languageAliases.get(normalized) ?? normalized +} diff --git a/packages/ts-ssg/src/mdx/highlight.ts b/packages/ts-ssg/src/mdx/highlight.ts index adc1ea0b..35ea765f 100644 --- a/packages/ts-ssg/src/mdx/highlight.ts +++ b/packages/ts-ssg/src/mdx/highlight.ts @@ -1,4 +1,5 @@ import { createHighlighter, type Highlighter } from 'shiki' +import { DEFAULT_CODE_LANGUAGES, normalizeCodeLanguage } from './codeLanguages' export interface MdxCodeHighlighter { codeToHtml: (code: string, lang?: string) => string | Promise @@ -20,38 +21,8 @@ export const DEFAULT_MDX_CODE_THEMES: MdxCodeThemes = { dark: 'github-dark', } -export const DEFAULT_MDX_CODE_LANGS: MdxCodeLangs = [ - 'bash', - 'sh', - 'shell', - 'javascript', - 'js', - 'typescript', - 'ts', - 'ts-tags', - 'jsx', - 'tsx', - 'json', - 'html', - 'css', - 'markdown', - 'mdx', - 'yaml', - 'yml', - 'toml', - 'diff', - 'csharp', - 'cpp', - 'c', - 'java', - 'python', - 'ruby', - 'go', - 'rust', - 'xml', -] +export const DEFAULT_MDX_CODE_LANGS: MdxCodeLangs = DEFAULT_CODE_LANGUAGES -const TYPESCRIPT_LANGS = new Set(['typescript', 'ts', 'cts', 'mts']) const TYPESCRIPT_WITH_TAGS_LANG = 'ts-tags' export async function createMdxHighlighter( @@ -78,11 +49,10 @@ export async function createMdxHighlighter( } function resolveLoadedLanguages(langs: string[]) { - const resolved = dedupe(langs.map(normalizeLanguageName)) + const resolved = dedupe(langs.map(normalizeCodeLanguage)) if ( - resolved.some((lang) => TYPESCRIPT_LANGS.has(lang)) && - !resolved.includes(TYPESCRIPT_WITH_TAGS_LANG) && - !resolved.includes('lit') + resolved.includes('typescript') && + !resolved.includes(TYPESCRIPT_WITH_TAGS_LANG) ) { resolved.push(TYPESCRIPT_WITH_TAGS_LANG) } @@ -90,16 +60,12 @@ function resolveLoadedLanguages(langs: string[]) { } function resolveHighlightLanguage(lang: string | undefined) { - const normalized = normalizeLanguageName(lang ?? 'text') - return TYPESCRIPT_LANGS.has(normalized) + const normalized = normalizeCodeLanguage(lang ?? 'text') + return normalized === 'typescript' ? TYPESCRIPT_WITH_TAGS_LANG : normalized || 'text' } -function normalizeLanguageName(value: string) { - return value.trim().toLowerCase() -} - function dedupe(values: string[]) { return Array.from(new Set(values.filter((value) => value.length > 0))) } diff --git a/packages/ts-ssg/src/mdx/highlightjs.ts b/packages/ts-ssg/src/mdx/highlightjs.ts index 9653ce0f..f0cac45c 100644 --- a/packages/ts-ssg/src/mdx/highlightjs.ts +++ b/packages/ts-ssg/src/mdx/highlightjs.ts @@ -1,25 +1,26 @@ import hljs, { type HLJSApi } from 'highlight.js' +import { normalizeCodeLanguage } from './codeLanguages' import type { MdxCodeHighlighter } from './highlight' -const languageAliases: Record = { - js: 'javascript', - ts: 'typescript', - sh: 'bash', - shell: 'bash', - yml: 'yaml', - md: 'markdown', -} +// Use highlight.js names, or the closest grammar for Shiki-only languages. +const hljsLanguages = new Map([ + ['jsonc', 'json'], + ['mdx', 'markdown'], + ['objective-c', 'objectivec'], + ['ts-tags', 'typescript'], +]) export function createHljsHighlighter( highlighter: HLJSApi = hljs, ): MdxCodeHighlighter { return { codeToHtml: (code: string, lang?: string) => { - const normalizedLang = normalizeLanguage(lang) - if (normalizedLang && highlighter.getLanguage(normalizedLang)) { + const normalizedLang = normalizeCodeLanguage(lang ?? '') + const highlightLang = hljsLanguages.get(normalizedLang) ?? normalizedLang + if (highlightLang && highlighter.getLanguage(highlightLang)) { const highlighted = highlighter.highlight(code, { - language: normalizedLang, + language: highlightLang, ignoreIllegals: true, }) return `
${highlighted.value}
` @@ -32,13 +33,6 @@ export function createHljsHighlighter( } } -function normalizeLanguage(lang: string | undefined): string | undefined { - if (!lang) return undefined - const normalized = lang.trim().toLowerCase() - if (!normalized) return undefined - return languageAliases[normalized] ?? normalized -} - function escapeHtml(value: string): string { return value .replaceAll('&', '&') diff --git a/packages/ts-ssg/src/mdx/imports.test.ts b/packages/ts-ssg/src/mdx/imports.test.ts index 8c421149..c29d57a9 100644 --- a/packages/ts-ssg/src/mdx/imports.test.ts +++ b/packages/ts-ssg/src/mdx/imports.test.ts @@ -12,6 +12,10 @@ describe('expandImports', () => { const files: Record = { 'guides/demo.ts': 'export const answer = 42\n', 'guides/styles.css': '.demo { color: red; }\n', + 'guides/example.PY': 'print(42)\n', + 'guides/example.cs': 'public class Example {}\n', + 'guides/example.ps1': 'Write-Host "Hello"\n', + 'guides/example.sql': 'SELECT 42;\n', 'shared/fence.md': 'Use a fence:\r\n\r\n```ts\r\nconst x = 1\r\n```\r\n', 'guides/data.unknown': 'plain', 'shared/_note.mdx': @@ -58,6 +62,17 @@ describe('expandImports', () => { ) }) + it.each([ + ['example.PY', 'python', 'print(42)'], + ['example.cs', 'csharp', 'public class Example {}'], + ['example.ps1', 'powershell', 'Write-Host "Hello"'], + ['example.sql', 'sql', 'SELECT 42;'], + ])('infers the language for %s', async (file, language, code) => { + expect(await expand(``)).toBe( + `\n\`\`\`${language}\n${code}\n\`\`\`\n`, + ) + }) + it('resolves src from the page, reports each import, and keeps line endings uniform', async () => { const imports: string[] = [] diff --git a/packages/ts-ssg/src/mdx/imports.ts b/packages/ts-ssg/src/mdx/imports.ts index f0c3f3a2..eb4abcfb 100644 --- a/packages/ts-ssg/src/mdx/imports.ts +++ b/packages/ts-ssg/src/mdx/imports.ts @@ -7,6 +7,7 @@ import remarkParse from 'remark-parse' import { unified } from 'unified' import { isSharedContentFile } from '../discover/content' import { isContentExt } from '../discover/contentExtensions' +import { LANGUAGE_BY_EXTENSION } from './codeLanguages' import { findMarkupTags } from './regorMarkup' const IMPORT_TAGS = { @@ -28,26 +29,6 @@ type ImportTagName = keyof typeof IMPORT_TAGS const IMPORT_TAG_NAMES = Object.keys(IMPORT_TAGS) as ImportTagName[] -const LANGUAGE_BY_EXTENSION: Record = { - '.cjs': 'javascript', - '.css': 'css', - '.cts': 'typescript', - '.html': 'html', - '.js': 'javascript', - '.json': 'json', - '.jsx': 'jsx', - '.md': 'markdown', - '.mdx': 'mdx', - '.mjs': 'javascript', - '.mts': 'typescript', - '.rmdx': 'mdx', - '.sh': 'bash', - '.ts': 'typescript', - '.tsx': 'tsx', - '.yaml': 'yaml', - '.yml': 'yaml', -} - const parser = unified().use(remarkParse).use(remarkGfm) export interface ImportOptions {