Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/core/cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ export interface CacheData {
const cacheFilePath = (): string => path.join(getCacheDir(), 'parsed.json');
const cacheMetaPath = (): string => path.join(getCacheDir(), 'meta.json');

const CACHE_VERSION = 30;
const CACHE_VERSION = 31;

/** Refuse to JSON.parse cache files beyond these sizes: a corrupted (or
* tampered) cache must degrade to a full re-parse, not OOM the host. */
Expand Down
35 changes: 35 additions & 0 deletions src/core/parser-claude.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { execSync } from 'child_process';
import { describe, it, expect } from 'vitest';
import { EditLocIndex } from './edit-loc-diff';
import { parseClaudeSessions } from './parser-claude';
import { MAX_FILE_SIZE } from './parser-shared';

/** os.tmpdir() on Windows often returns 8.3 short names (e.g. TAMASB~1)
* that don't match readdirSync output. Resolve to the long form so
Expand Down Expand Up @@ -646,3 +647,37 @@ describe('parseClaudeSessions', () => {
fs.rmSync(cwd, { recursive: true, force: true });
});
});

describe('parseClaudeSessions: large transcripts', () => {
it('parses a transcript over the size limit by leaving out its API request records', { timeout: 60_000 }, () => {
const blob = 'x'.repeat(1024 * 1024);
const apiRequests = Array.from({ length: Math.ceil(MAX_FILE_SIZE / blob.length) + 2 }, () => ({ type: 'api-request', body: blob }));
withProjectsDir('s.jsonl', [
makeUser('Why?', '2025-06-15T10:00:00Z'),
...apiRequests,
makeAssistant('Because.'),
], projectsDir => {
const file = path.join(projectsDir, '-Users-me-proj', 's.jsonl');
expect(fs.statSync(file).size).toBeGreaterThan(MAX_FILE_SIZE);
const requests = parseClaudeSessions(projectsDir)[0].sessions[0].requests;
expect(requests.map(request => [request.messageText, request.responseText])).toEqual([['Why?', 'Because.']]);
});
});

it('leaves out API request records whatever the key order, and keeps a prompt that mentions one', { timeout: 60_000 }, () => {
const blob = 'x'.repeat(1024 * 1024);
const apiRequests = Array.from({ length: Math.ceil(MAX_FILE_SIZE / blob.length) + 2 }, (_, i) => (
{ timestamp: '2025-06-15T10:00:01Z', uuid: `req-${i}`, type: 'api-request', body: blob }));
const mention = 'Why does {"type":"api-request"} show up?';
withProjectsDir('s.jsonl', [
makeUser(mention, '2025-06-15T10:00:00Z'),
...apiRequests,
makeAssistant('Because.'),
], projectsDir => {
const file = path.join(projectsDir, '-Users-me-proj', 's.jsonl');
expect(fs.statSync(file).size).toBeGreaterThan(MAX_FILE_SIZE);
const requests = parseClaudeSessions(projectsDir)[0].sessions[0].requests;
expect(requests.map(request => [request.messageText, request.responseText])).toEqual([[mention, 'Because.']]);
});
});
});
38 changes: 26 additions & 12 deletions src/core/parser-claude.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import * as fs from 'fs';
import * as path from 'path';
import { createHash } from 'crypto';
import { Session, SessionRequest } from './types';
import { assertTrustedPath, readFileSafe, createRequest, createSession, detectDevcontainerFromRequests, extractSkillNameFromPath } from './parser-shared';
import { assertTrustedPath, MAX_FILE_SIZE, createRequest, createSession, detectDevcontainerFromRequests, extractSkillNameFromPath } from './parser-shared';
import { extractReasoningEffortFromModelId } from './helpers';
import { warnCore } from './log';
import { EditLocIndex } from './edit-loc-diff';
Expand Down Expand Up @@ -195,13 +195,31 @@ function parseClaudeLine(rawLine: string): ClaudeLine | null {
}
}

function parseClaudeLines(raw: string): ClaudeLine[] {
/**
* Raw API request records: most of a large transcript's bytes, and nothing the parser reads. Matched wherever `type`
* sits among the keys. Quotes inside JSON string values are escaped, so a prompt that mentions the text cannot match.
*/
const CLAUDE_API_REQUEST_LINE = /"type"\s*:\s*"api-request/;

/**
* Streams a transcript's lines, leaving out raw API request records. Stops with a warning once the kept lines pass
* MAX_FILE_SIZE, so a huge transcript is parsed up to that point instead of skipped whole.
*/
function readClaudeLines(filePath: string): ClaudeLine[] {
const lines: ClaudeLine[] = [];
for (const rawLine of raw.split('\n')) {
if (!rawLine.trim()) continue;
let keptBytes = 0;
let truncated = false;
forEachJsonlLine(filePath, rawLine => {
if (!rawLine.trim() || CLAUDE_API_REQUEST_LINE.test(rawLine)) return;
keptBytes += rawLine.length;
if (keptBytes > MAX_FILE_SIZE) {
truncated = true;
return;
}
const parsed = parseClaudeLine(rawLine);
if (parsed) lines.push(parsed);
}
if (parsed && !parsed.type.startsWith('api-request')) lines.push(parsed);
}, () => truncated);
if (truncated) warnCore('parser-claude', `Transcript over ${MAX_FILE_SIZE / 1024 / 1024} MB without API requests; parsed up to there: ${filePath}`);
return lines;
}

Expand Down Expand Up @@ -821,16 +839,12 @@ export function parseClaudeSessionFile(
{ editLocIndex, subagentFile = false }: { editLocIndex?: EditLocIndex; subagentFile?: boolean } = {},
): Session | null {
assertTrustedPath(filePath);
let raw: string;
let lines: ClaudeLine[];
try {
const content = readFileSafe(filePath);
if (content === null) return null;
raw = content;
lines = readClaudeLines(filePath);
} catch {
return null;
}

const lines = parseClaudeLines(raw);
if (lines.length === 0) return null;

const sessionId = lines[0].sessionId || path.basename(filePath, '.jsonl');
Expand Down