diff --git a/src/core/cache.ts b/src/core/cache.ts index 3283b1d7..f7cd6eb2 100644 --- a/src/core/cache.ts +++ b/src/core/cache.ts @@ -130,7 +130,7 @@ export interface CacheData { const cacheFilePath = (): string => path.join(getCacheDir(), 'parsed.json'); const cacheMetaPath = (): string => path.join(getCacheDir(), 'meta.json'); -const CACHE_VERSION = 30; +const CACHE_VERSION = 31; /** Refuse to JSON.parse cache files beyond these sizes: a corrupted (or * tampered) cache must degrade to a full re-parse, not OOM the host. */ diff --git a/src/core/parser-claude.test.ts b/src/core/parser-claude.test.ts index a38a20bd..1111849c 100644 --- a/src/core/parser-claude.test.ts +++ b/src/core/parser-claude.test.ts @@ -14,6 +14,7 @@ import { execSync } from 'child_process'; import { describe, it, expect } from 'vitest'; import { EditLocIndex } from './edit-loc-diff'; import { parseClaudeSessions } from './parser-claude'; +import { MAX_FILE_SIZE } from './parser-shared'; /** os.tmpdir() on Windows often returns 8.3 short names (e.g. TAMASB~1) * that don't match readdirSync output. Resolve to the long form so @@ -646,3 +647,37 @@ describe('parseClaudeSessions', () => { fs.rmSync(cwd, { recursive: true, force: true }); }); }); + +describe('parseClaudeSessions: large transcripts', () => { + it('parses a transcript over the size limit by leaving out its API request records', { timeout: 60_000 }, () => { + const blob = 'x'.repeat(1024 * 1024); + const apiRequests = Array.from({ length: Math.ceil(MAX_FILE_SIZE / blob.length) + 2 }, () => ({ type: 'api-request', body: blob })); + withProjectsDir('s.jsonl', [ + makeUser('Why?', '2025-06-15T10:00:00Z'), + ...apiRequests, + makeAssistant('Because.'), + ], projectsDir => { + const file = path.join(projectsDir, '-Users-me-proj', 's.jsonl'); + expect(fs.statSync(file).size).toBeGreaterThan(MAX_FILE_SIZE); + const requests = parseClaudeSessions(projectsDir)[0].sessions[0].requests; + expect(requests.map(request => [request.messageText, request.responseText])).toEqual([['Why?', 'Because.']]); + }); + }); + + it('leaves out API request records whatever the key order, and keeps a prompt that mentions one', { timeout: 60_000 }, () => { + const blob = 'x'.repeat(1024 * 1024); + const apiRequests = Array.from({ length: Math.ceil(MAX_FILE_SIZE / blob.length) + 2 }, (_, i) => ( + { timestamp: '2025-06-15T10:00:01Z', uuid: `req-${i}`, type: 'api-request', body: blob })); + const mention = 'Why does {"type":"api-request"} show up?'; + withProjectsDir('s.jsonl', [ + makeUser(mention, '2025-06-15T10:00:00Z'), + ...apiRequests, + makeAssistant('Because.'), + ], projectsDir => { + const file = path.join(projectsDir, '-Users-me-proj', 's.jsonl'); + expect(fs.statSync(file).size).toBeGreaterThan(MAX_FILE_SIZE); + const requests = parseClaudeSessions(projectsDir)[0].sessions[0].requests; + expect(requests.map(request => [request.messageText, request.responseText])).toEqual([[mention, 'Because.']]); + }); + }); +}); diff --git a/src/core/parser-claude.ts b/src/core/parser-claude.ts index 5281138d..eee272c6 100644 --- a/src/core/parser-claude.ts +++ b/src/core/parser-claude.ts @@ -19,7 +19,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { createHash } from 'crypto'; import { Session, SessionRequest } from './types'; -import { assertTrustedPath, readFileSafe, createRequest, createSession, detectDevcontainerFromRequests, extractSkillNameFromPath } from './parser-shared'; +import { assertTrustedPath, MAX_FILE_SIZE, createRequest, createSession, detectDevcontainerFromRequests, extractSkillNameFromPath } from './parser-shared'; import { extractReasoningEffortFromModelId } from './helpers'; import { warnCore } from './log'; import { EditLocIndex } from './edit-loc-diff'; @@ -195,13 +195,31 @@ function parseClaudeLine(rawLine: string): ClaudeLine | null { } } -function parseClaudeLines(raw: string): ClaudeLine[] { +/** + * Raw API request records: most of a large transcript's bytes, and nothing the parser reads. Matched wherever `type` + * sits among the keys. Quotes inside JSON string values are escaped, so a prompt that mentions the text cannot match. + */ +const CLAUDE_API_REQUEST_LINE = /"type"\s*:\s*"api-request/; + +/** + * Streams a transcript's lines, leaving out raw API request records. Stops with a warning once the kept lines pass + * MAX_FILE_SIZE, so a huge transcript is parsed up to that point instead of skipped whole. + */ +function readClaudeLines(filePath: string): ClaudeLine[] { const lines: ClaudeLine[] = []; - for (const rawLine of raw.split('\n')) { - if (!rawLine.trim()) continue; + let keptBytes = 0; + let truncated = false; + forEachJsonlLine(filePath, rawLine => { + if (!rawLine.trim() || CLAUDE_API_REQUEST_LINE.test(rawLine)) return; + keptBytes += rawLine.length; + if (keptBytes > MAX_FILE_SIZE) { + truncated = true; + return; + } const parsed = parseClaudeLine(rawLine); - if (parsed) lines.push(parsed); - } + if (parsed && !parsed.type.startsWith('api-request')) lines.push(parsed); + }, () => truncated); + if (truncated) warnCore('parser-claude', `Transcript over ${MAX_FILE_SIZE / 1024 / 1024} MB without API requests; parsed up to there: ${filePath}`); return lines; } @@ -821,16 +839,12 @@ export function parseClaudeSessionFile( { editLocIndex, subagentFile = false }: { editLocIndex?: EditLocIndex; subagentFile?: boolean } = {}, ): Session | null { assertTrustedPath(filePath); - let raw: string; + let lines: ClaudeLine[]; try { - const content = readFileSafe(filePath); - if (content === null) return null; - raw = content; + lines = readClaudeLines(filePath); } catch { return null; } - - const lines = parseClaudeLines(raw); if (lines.length === 0) return null; const sessionId = lines[0].sessionId || path.basename(filePath, '.jsonl');