feat(explore): markdown index (#361) with a section-first doc tier

__tests__/explore-doc-tier-named.test.ts

@@ -0,0 +1,127 @@
+/**
+ * The doc tier's naming path: a query that IS a document's name.
+ *
+ * `collectDocSeeds` honoured `named` at the file gate and ignored it at the
+ * section gate, so `CONTRIBUTING.md` found the file and then dropped it for
+ * want of a scoring line — the user typed a filename and got nothing. Three
+ * rules combined to make every section score 0 for such a query: a term
+ * appearing in the file path is weighted 0 (and for a bare filename that is
+ * the only term), `lineScore` needs two distinct terms on one line, and
+ * `coveredHeading` needs a term of non-zero weight.
+ *
+ * A fourth rule made some headings unreachable by any query at all:
+ * `coveredHeading` demanded every significant heading word be covered, but
+ * DOC_QUERY_NOISE words are stripped from the query, so a heading containing
+ * "repo" could never be covered.
+ *
+ * These are retrieval assertions, not extraction ones — the misses that
+ * motivated them were found by querying a real 84-file repo, which is exactly
+ * what no unit test here was doing.
+ */
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import CodeGraph from '../src/index';
+import { ToolHandler } from '../src/mcp/tools';
+
+let dir: string;
+let cg: CodeGraph;
+
+async function explore(query: string): Promise<string> {
+  const res = await new ToolHandler(cg).execute('codegraph_explore', { query });
+  return res.content?.[0]?.text ?? '';
+}
+
+/** The response renders a source section for `file`. */
+const hasSection = (response: string, file: string): boolean =>
+  response.includes('**`' + file + '`');
+
+const CONTRIBUTING = `# Project Contributing Guide
+
+Some preamble that names no section.
+
+## Repo Setup
+
+Clone it and install dependencies.
+
+## Cloning the repo on Windows
+
+Enable symlinks and long paths before cloning.
+
+## Ignoring commits when running git blame
+
+Use the ignore-revs file.
+`;
+
+const README = `# Widget
+
+A widget.
+
+## Installation
+
+Install the widget.
+
+## Usage
+
+Use the widget.
+`;
+
+beforeAll(async () => {
+  dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-doc-named-'));
+  fs.mkdirSync(path.join(dir, 'src'), { recursive: true });
+  fs.mkdirSync(path.join(dir, 'test', 'fixtures'), { recursive: true });
+  fs.writeFileSync(path.join(dir, 'CONTRIBUTING.md'), CONTRIBUTING);
+  fs.writeFileSync(path.join(dir, 'README.md'), README);
+  // A fixture copy, which DOC_LOW_PATH exists to keep out of results.
+  fs.writeFileSync(path.join(dir, 'test', 'fixtures', 'README.md'), README);
+  // Code, so the index is not markdown-only and code queries have somewhere to go.
+  fs.writeFileSync(
+    path.join(dir, 'src', 'widget.ts'),
+    `export function createWidget(size: number): number { return size * 2; }\n` +
+      `export function resizeWidget(w: number): number { return createWidget(w); }\n`
+  );
+  cg = CodeGraph.initSync(dir);
+  await cg.indexAll();
+}, 180_000);
+
+afterAll(() => {
+  cg?.destroy();
+  if (dir && fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true });
+});
+
+describe('a query that names a markdown file', () => {
+  it('renders the file it names, spelled with the extension', async () => {
+    const out = await explore('CONTRIBUTING.md');
+    expect(hasSection(out, 'CONTRIBUTING.md')).toBe(true);
+  });
+
+  it('renders the file it names, spelled as the bare stem', async () => {
+    const out = await explore('readme');
+    expect(hasSection(out, 'README.md')).toBe(true);
+  });
+
+  it('does not let a bare stem surface a fixture copy', async () => {
+    // The stem match unlocks the two-hit gate and the section fallback, but
+    // deliberately not the DOC_LOW_PATH bypass — that stays the privilege of
+    // an explicitly spelled path.
+    const out = await explore('readme');
+    expect(hasSection(out, 'test/fixtures/README.md')).toBe(false);
+  });
+});
+
+describe('a heading containing a DOC_QUERY_NOISE word', () => {
+  it('is reachable, though "repo" can never appear among the query terms', async () => {
+    const out = await explore('the contributing guide for windows');
+    expect(hasSection(out, 'CONTRIBUTING.md')).toBe(true);
+    expect(out).toContain('Cloning the repo on Windows');
+  });
+});
+
+describe('the DOC_WORD gate still declines what it should', () => {
+  it('a question with no doc word pulls no markdown', async () => {
+    const out = await explore('how do i resize a widget');
+    expect(hasSection(out, 'CONTRIBUTING.md')).toBe(false);
+    expect(hasSection(out, 'README.md')).toBe(false);
+  });
+});

__tests__/extraction.test.ts

@@ -103,6 +103,12 @@ describe('Language Detection', () => {
     expect(detectLanguage('stdio.h', '#ifndef STDIO_H\nvoid printf();\n#endif\n')).toBe('c');
   });
 
+  it('should detect Markdown files', () => {
+    expect(detectLanguage('README.md')).toBe('markdown');
+    expect(detectLanguage('docs/guide.markdown')).toBe('markdown');
+    expect(detectLanguage('docs/page.mdx')).toBe('markdown');
+  });
+
   it('should detect Metal shader files as C++ (#1121)', () => {
     expect(detectLanguage('Shaders.metal')).toBe('cpp');
     expect(isSourceFile('Renderer/Shaders.metal')).toBe(true);
@@ -251,11 +257,218 @@ describe('Language Support', () => {
     expect(languages).toContain('swift');
     expect(languages).toContain('kotlin');
     expect(languages).toContain('dart');
+    expect(languages).toContain('markdown');
     expect(languages).toContain('solidity');
     expect(languages).toContain('nix');
   });
 });
 
+describe('Markdown Extraction', () => {
+  it('should extract headings, links, and shell script references', () => {
+    const markdown = `# Project Guide
+
+See [Setup](docs/setup.md#install) and scripts/release.mjs.
+
+## Release
+
+\`\`\`bash
+npm run build
+node scripts/release.mjs
+\`\`\`
+`;
+
+    const result = extractFromSource('README.md', markdown);
+
+    const fileNode = result.nodes.find((n) => n.kind === 'file');
+    expect(fileNode).toMatchObject({
+      name: 'README.md',
+      language: 'markdown',
+    });
+
+    const headings = result.nodes.filter((n) => n.kind === 'module');
+    expect(headings.map((n) => n.name)).toContain('Project Guide');
+    expect(headings.map((n) => n.name)).toContain('Release');
+
+    const commandNode = result.nodes.find((n) => n.kind === 'function' && n.signature === 'node scripts/release.mjs');
+    expect(commandNode).toBeDefined();
+
+    expect(result.unresolvedReferences).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          referenceName: 'docs/setup.md#install',
+          referenceKind: 'imports',
+          language: 'markdown',
+        }),
+        expect.objectContaining({
+          referenceName: 'scripts/release.mjs',
+          referenceKind: 'calls',
+          language: 'markdown',
+        }),
+      ])
+    );
+  });
+
+  it('should extract structured table rows and file-symbol references from Markdown', () => {
+    const markdown = `# Maintenance Guide
+
+## Phase 4
+
+| Template | CLI Entry | Dispatcher | Implementation |
+| --- | --- | --- | --- |
+| P4-S1 | \`python "{script_path}" p4 "{csv_file}" s1 "{conditions_or_-}" "{probe_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage1\` |
+| P4-S2 | \`python "{script_path}" p4 "{csv_file}" s2 "{stage1_rows}" "{condition_or_-}" "{detail_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage2\` |
+
+- P4-FLOW changes must inspect \`scripts/csv_search.py::run_p4\`.
+`;
+
+    const result = extractFromSource('phases/phase4.md', markdown);
+
+    const tableRows = result.nodes.filter((n) => n.kind === 'constant' && n.qualifiedName.includes('table-row'));
+    expect(tableRows.map((n) => n.name)).toEqual(expect.arrayContaining(['P4-S1', 'P4-S2']));
+
+    const p4s1 = tableRows.find((n) => n.name === 'P4-S1');
+    expect(p4s1?.signature).toContain('Template: P4-S1');
+    expect(p4s1?.signature).toContain('Dispatcher: scripts/csv_search.py::run_p4');
+
+    const commandNode = result.nodes.find((n) =>
+      n.kind === 'function' &&
+      n.language === 'markdown' &&
+      n.signature?.includes('python "{script_path}" p4')
+    );
+    expect(commandNode).toBeDefined();
+
+    expect(result.unresolvedReferences).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          referenceName: 'phases/scripts/csv_search.py::run_p4',
+          referenceKind: 'references',
+          language: 'markdown',
+        }),
+        expect.objectContaining({
+          referenceName: 'phases/scripts/csv_search.py::_p4_stage1',
+          referenceKind: 'references',
+          language: 'markdown',
+        }),
+      ])
+    );
+  });
+
+  it('should keep structured blocks after fences containing a different fence marker', () => {
+    const markdown = `# Runbook
+
+\`\`\`text
+~~~~
+\`\`\`
+
+- POST-FENCE references \`src/auth.ts::login\`.
+
+| Key | Target |
+| --- | --- |
+| POST-TABLE | \`src/auth.ts::login\` |
+`;
+
+    const result = extractFromSource('docs/runbook.md', markdown);
+    const constants = result.nodes.filter((n) => n.kind === 'constant');
+
+    expect(constants).toEqual(expect.arrayContaining([
+      expect.objectContaining({ docstring: 'POST-FENCE references src/auth.ts::login.' }),
+      expect.objectContaining({ name: 'POST-TABLE' }),
+    ]));
+  });
+
+  it('indexes Setext (underline) headings and skips frontmatter / code fences', () => {
+    const markdown = `---
+title: Config Doc
+---
+
+Architecture Overview
+=====================
+
+Intro paragraph for the overview.
+
+Routing Layer
+-------------
+
+\`\`\`md
+Not A Heading
+=============
+\`\`\`
+`;
+
+    const result = extractFromSource('docs/arch.md', markdown);
+    const headings = result.nodes.filter((n) => n.kind === 'module');
+    const byName = new Map(headings.map((h) => [h.name, h]));
+
+    // Setext H1 (===) and H2 (---) become module nodes.
+    expect(byName.get('Architecture Overview')?.signature).toBe('# Architecture Overview');
+    expect(byName.get('Routing Layer')?.signature).toBe('## Routing Layer');
+    // Frontmatter `title:` (above the closing `---`) is NOT a heading, and a
+    // setext-looking line inside a code fence is ignored.
+    expect(byName.has('title: Config Doc')).toBe(false);
+    expect(byName.has('Not A Heading')).toBe(false);
+  });
+
+  it('builds a deterministic, compact file digest (intro + key references)', () => {
+    const markdown = `# Release Runbook
+
+This runbook explains how to cut a release.
+
+See [setup](docs/setup.md#install) and run \`scripts/release.mjs\`.
+It dispatches \`scripts/csv_search.py::run_p4\`.
+`;
+
+    const result = extractFromSource('RUNBOOK.md', markdown);
+    const fileNode = result.nodes.find((n) => n.kind === 'file');
+
+    expect(fileNode?.docstring).toBeDefined();
+    const digest = fileNode!.docstring!;
+    // Intro is the first prose line, not the heading or a link blob.
+    expect(digest).toContain('This runbook explains how to cut a release.');
+    // Key referenced files/symbols are surfaced, compacted to basenames.
+    expect(digest).toContain('refs:');
+    expect(digest).toContain('setup.md#install');
+    expect(digest).toContain('release.mjs');
+    expect(digest).toContain('csv_search.py::run_p4');
+    // Short enough to show in node details (the < 200 char detail gate).
+    expect(digest.length).toBeLessThan(200);
+  });
+});
+
+describe('Code to Markdown Reference Extraction', () => {
+  it('should extract Markdown path references from code string literals', () => {
+    const code = `
+export const GUIDE = '../docs/guide.md';
+
+export function loadDocs() {
+  return fs.readFileSync('../docs/guide.md#install', 'utf8');
+}
+`;
+
+    const result = extractFromSource('src/load-docs.ts', code);
+    const loadDocs = result.nodes.find((n) => n.kind === 'function' && n.name === 'loadDocs');
+    const guideConstant = result.nodes.find((n) => n.kind === 'constant' && n.name === 'GUIDE');
+
+    expect(loadDocs).toBeDefined();
+    expect(guideConstant).toBeDefined();
+    expect(result.unresolvedReferences).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          fromNodeId: loadDocs!.id,
+          referenceName: 'docs/guide.md#install',
+          referenceKind: 'references',
+          language: 'typescript',
+        }),
+        expect.objectContaining({
+          fromNodeId: guideConstant!.id,
+          referenceName: 'docs/guide.md',
+          referenceKind: 'references',
+          language: 'typescript',
+        }),
+      ])
+    );
+  });
+});
+
 describe('Nix Extraction', () => {
   it('should distinguish Nix variable and function bindings', () => {
     const code = `

__tests__/fixtures/kernel-parity/Torture.cs

@@ -179,3 +179,25 @@ namespace Torture.Beta
 {
     public class Other { }
 }
+
+// Markdown path references: code -> documentation edges. Every shape the
+// normalizer branches on, so the two arms have to agree about the rejections
+// (URL, escape above the root) as well as the emissions.
+namespace Torture.Markdown
+{
+    public class MarkdownPaths
+    {
+        public const string Guide = "../docs/guide.md#install";
+        public const string Bare = "README.md";
+        public const string Rooted = "/docs/rooted.md";
+        public const string Escapes = "../../../../outside.md";
+        public const string Remote = "https://example.com/remote.md";
+        public const string Queried = "./notes.md?raw=1#top";
+        public const string TwoInOne = "see a.md and also sub/b.mdx";
+
+        public void Load()
+        {
+            LoadDoc("docs/deep/nested.markdown");
+        }
+    }
+}