diff --git a/src/core/agent-compaction.ts b/src/core/agent-compaction.ts new file mode 100644 index 00000000..4dfd2218 --- /dev/null +++ b/src/core/agent-compaction.ts @@ -0,0 +1,42 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/* Claude Code and Codex compactions, in the shape of the Copilot CLI events (`cli-compaction.ts`). + * + * Claude Code writes a `system` line with subtype `compact_boundary` and `compactMetadata` + * (`trigger`, `preTokens`, `postTokens`, `durationMs`). + * Codex logs no sizes: a compaction shows as a `compacted` line, a `compaction` response item, a + * `context_compacted` event or a `ContextCompaction` item, often several of them for one compaction. The size + * before is the prompt of the last model call and the limit its `model_context_window`. + * Only numbers and enum values are kept, never the summary. */ + +import type { CompactionEvent, SessionRequest } from './types'; +import { readCliCompaction } from './cli-compaction'; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Map a Claude Code `compactMetadata` to a compaction event. */ +export function readClaudeCompaction(metadata: unknown): CompactionEvent { + const data = isRecord(metadata) ? metadata : {}; + return readCliCompaction({ + contextLengthBefore: data.preTokens, + postCompactionTokens: data.postTokens, + trigger: data.trigger, + durationMs: data.durationMs, + }); +} + +/** A Codex compaction: the context of the last model call before it, and that call's limit when logged. */ +export function codexCompaction(contextBefore: number, tokenLimit: number | null): CompactionEvent { + return readCliCompaction({ contextLengthBefore: contextBefore, ...(tokenLimit ? { tokenLimit } : {}) }); +} + +/** The request fields for a turn's compactions: `compaction` is the first, `compactions` all of them when several. */ +export function requestCompactionFields(compactions: CompactionEvent[]): Partial> { + if (compactions.length === 0) return {}; + return { compaction: compactions[0], ...(compactions.length > 1 ? { compactions: [...compactions] } : {}) }; +} diff --git a/src/core/analyzer-context.ts b/src/core/analyzer-context.ts index 46c1798b..f5aa0984 100644 --- a/src/core/analyzer-context.ts +++ b/src/core/analyzer-context.ts @@ -500,6 +500,9 @@ export class ContextAnalyzer extends AnalyzerBase { const window = sessionContextWindow(s); for (let i = 0; i < s.requests.length; i++) { for (const compaction of requestCompactions(s.requests[i])) { + // Claude Code logs no window, and an estimate sized from the session's own peak + // puts every compaction near "full". Only a logged limit can show it. + if (compaction.tokenLimit == null && s.harness === 'Claude' && requestContextLimit(s.requests[i]) == null) continue; const limit = compaction.tokenLimit ?? window.perRequest[i]; if (compaction.contextLengthBefore > 0 && (compaction.contextLengthBefore / limit) * 100 > 85) { events.push({ session: s, tokensBefore: compaction.contextLengthBefore }); @@ -606,10 +609,12 @@ export class ContextAnalyzer extends AnalyzerBase { }); } - // 4. Runaway Context Growth — sessions where token count grows steadily without stabilizing - const runawayEntries: { session: Session; first: number; last: number }[] = []; + // 4. Runaway Context Growth — sessions where token count grows steadily without stabilizing. A compaction + // resets the context, so only the requests after the session's last compaction count. + const runawayEntries: { session: Session; first: number; last: number; requests: number }[] = []; for (const s of sessions) { - const sessionTokens = this.estimateSessionTokens(s); + const start = requestsAfterLastCompaction(s); + const sessionTokens = this.estimateSessionTokens(s).slice(start); if (sessionTokens.length < CONTEXT_GROWING_SESSION_MIN_REQS) continue; let increasing = 0; for (let i = 1; i < sessionTokens.length; i++) { @@ -617,7 +622,12 @@ export class ContextAnalyzer extends AnalyzerBase { } const growthRate = increasing / (sessionTokens.length - 1); if (growthRate > CONTEXT_GROWING_SESSION_GROWTH_RATE) { - runawayEntries.push({ session: s, first: sessionTokens[0], last: sessionTokens[sessionTokens.length - 1] }); + runawayEntries.push({ + session: s, + first: sessionTokens[0], + last: sessionTokens[sessionTokens.length - 1], + requests: start === 0 ? s.requestCount : sessionTokens.length, + }); } } if (runawayEntries.length > 0) { @@ -628,7 +638,7 @@ export class ContextAnalyzer extends AnalyzerBase { message: `${Math.round(r.first / 1000)}K \u2192 ${Math.round(r.last / 1000)}K tokens`, model: '', kind: 'session' as const, - stats: { startK: Math.round(r.first / 1000), endK: Math.round(r.last / 1000), requests: r.session.requestCount }, + stats: { startK: Math.round(r.first / 1000), endK: Math.round(r.last / 1000), requests: r.requests }, })); patterns.push({ id: 'runaway-context', @@ -969,6 +979,15 @@ export class ContextAnalyzer extends AnalyzerBase { } } +/** Index of the first request after the session's last successful compaction; 0 when it never compacted. The + * request that logged the compaction is left out: its last prompt can be from either side of it. */ +function requestsAfterLastCompaction(session: Session): number { + for (let i = session.requests.length - 1; i >= 0; i--) { + if (requestCompactions(session.requests[i]).some(compaction => compaction.outcome !== 'failed')) return i + 1; + } + return 0; +} + /** Adds the timestamps of the session's requests that carry composition but no per-request tokens. They add a * range for the composition the tab charts, but they aren't per-request token data. A request with tokens, the * prompt of its last model call included (`requestContextTokens`), is already counted. */ diff --git a/src/core/cache.ts b/src/core/cache.ts index 3283b1d7..f7cd6eb2 100644 --- a/src/core/cache.ts +++ b/src/core/cache.ts @@ -130,7 +130,7 @@ export interface CacheData { const cacheFilePath = (): string => path.join(getCacheDir(), 'parsed.json'); const cacheMetaPath = (): string => path.join(getCacheDir(), 'meta.json'); -const CACHE_VERSION = 30; +const CACHE_VERSION = 31; /** Refuse to JSON.parse cache files beyond these sizes: a corrupted (or * tampered) cache must degrade to a full re-parse, not OOM the host. */ diff --git a/src/core/context-management.test.ts b/src/core/context-management.test.ts index 4af609c3..ddf4fec7 100644 --- a/src/core/context-management.test.ts +++ b/src/core/context-management.test.ts @@ -600,6 +600,19 @@ describe('anti-pattern: context-bloat', () => { expect(bloat!.occurrences).toBe(1); }); + it('does not flag amnesia from an estimated Claude Code window', () => { + const compacted = (harness: string) => { + const requests = flatTokenRequests(CONTEXT_MIN_TOKEN_REQUESTS + 1, 400_000); + requests[3].compaction = { + mode: 'full', numRounds: 1, numRoundsSinceLastSummarization: 1, contextLengthBefore: 400_000, + durationMs: 1, model: 'm', outcome: 'success', + }; + return getCtxMgmt([sess({ harness, requests })]).antiPatterns.find(p => p.id === 'context-amnesia'); + }; + expect(compacted('Claude')).toBeUndefined(); + expect(compacted('Codex')).toBeDefined(); + }); + it('does not trigger for optimal sessions', () => { const sessions = [ sess({ requests: flatTokenRequests(CONTEXT_MIN_TOKEN_REQUESTS + 1, 30_000) }), @@ -751,6 +764,35 @@ describe('anti-pattern: runaway-context', () => { const runaway = data.antiPatterns.find(p => p.id === 'runaway-context'); expect(runaway).toBeUndefined(); }); + + it('treats a compaction as a reset: growth before it does not count', () => { + // Steady growth for 10 requests, a compaction, then 4 more growing requests: too few after the reset. + const reqs = [ + ...linearTokenRequests(10, 10_000, 5_000), + ...linearTokenRequests(4, 8_000, 2_000, BASE_TS + 10 * 60_000).map((r, i) => ({ ...r, requestId: `after-${i}` })), + ]; + reqs[9] = { ...reqs[9], compaction: compaction({ contextLengthBefore: 55_000, contextLengthAfter: 6_000, trigger: 'auto' }) }; + const data = getCtxMgmt([sess({ requests: reqs })]); + expect(data.antiPatterns.find(p => p.id === 'runaway-context')).toBeUndefined(); + }); + + it('still flags steady growth after the last compaction, measured from the reset', () => { + const before = linearTokenRequests(3, 50_000, 10_000); + before[2] = { ...before[2], compaction: compaction({ contextLengthBefore: 70_000, contextLengthAfter: 8_000 }) }; + const after = linearTokenRequests(CONTEXT_GROWING_SESSION_MIN_REQS + 1, 10_000, 5_000, BASE_TS + 3 * 60_000) + .map((r, i) => ({ ...r, requestId: `after-${i}` })); + const data = getCtxMgmt([sess({ requests: [...before, ...after] })]); + const runaway = data.antiPatterns.find(p => p.id === 'runaway-context'); + expect(runaway?.occurrences).toBe(1); + expect(runaway?.details?.[0].stats).toEqual({ startK: 10, endK: 50, requests: CONTEXT_GROWING_SESSION_MIN_REQS + 1 }); + }); + + it('ignores a failed compaction', () => { + const reqs = linearTokenRequests(CONTEXT_GROWING_SESSION_MIN_REQS + 2, 10_000, 5_000); + reqs[5] = { ...reqs[5], compaction: compaction({ outcome: 'failed' }) }; + const data = getCtxMgmt([sess({ requests: reqs })]); + expect(data.antiPatterns.find(p => p.id === 'runaway-context')?.occurrences).toBe(1); + }); }); /* ── Tips generation ─────────────────────────────────────────────── */ diff --git a/src/core/parser-claude-compaction.test.ts b/src/core/parser-claude-compaction.test.ts new file mode 100644 index 00000000..50d69393 --- /dev/null +++ b/src/core/parser-claude-compaction.test.ts @@ -0,0 +1,82 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/* Claude Code compactions: a `system` line with subtype `compact_boundary` and `compactMetadata`. The + * fixtures are synthetic, in the shape Claude Code writes. */ + +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { describe, expect, it } from 'vitest'; +import { parseClaudeSessions } from './parser-claude'; +import { requestCompactions } from './context-window'; +import { SessionSchema } from './schemas'; +import type { Session } from './types'; + +const SESSION = 'sess-compact'; + +function parseLines(lines: object[]): Session { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'claude-compaction-test-')); + const projDir = path.join(root, 'projects', '-Users-me-proj'); + fs.mkdirSync(projDir, { recursive: true }); + fs.writeFileSync(path.join(projDir, `${SESSION}.jsonl`), lines.map(l => JSON.stringify(l)).join('\n'), 'utf-8'); + try { + return parseClaudeSessions(path.join(root, 'projects'))[0].sessions[0]; + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +} + +function user(text: string, ts: string, extra: Record = {}): object { + return { type: 'user', timestamp: ts, sessionId: SESSION, cwd: '/Users/me/proj', ...extra, message: { role: 'user', content: [{ type: 'text', text }] } }; +} + +function assistant(id: string, ts: string, input: number): object { + return { + type: 'assistant', timestamp: ts, sessionId: SESSION, + message: { id, role: 'assistant', model: 'claude-sonnet-4-6', content: [{ type: 'text', text: 'done' }], usage: { input_tokens: input, output_tokens: 10 } }, + }; +} + +function boundary(ts: string, metadata: Record): object { + return { type: 'system', subtype: 'compact_boundary', content: 'Conversation compacted', timestamp: ts, sessionId: SESSION, compactMetadata: metadata }; +} + +describe('Claude Code compaction', () => { + it('records a compact_boundary on the request it was logged in, with its sizes and trigger', () => { + const session = parseLines([ + user('first task', '2026-11-01T10:00:00Z'), + assistant('m1', '2026-11-01T10:00:05Z', 300_000), + boundary('2026-11-01T10:00:06Z', { trigger: 'auto', preTokens: 300_000, postTokens: 12_000, durationMs: 40 }), + user('This session is being continued from a previous conversation.', '2026-11-01T10:00:07Z', { isCompactSummary: true }), + assistant('m2', '2026-11-01T10:00:09Z', 12_000), + user('next task', '2026-11-01T10:01:00Z'), + assistant('m3', '2026-11-01T10:01:05Z', 15_000), + ]); + expect(session.requests).toHaveLength(3); + expect(session.requests[0].compaction).toMatchObject({ + contextLengthBefore: 300_000, contextLengthAfter: 12_000, trigger: 'auto', durationMs: 40, outcome: 'success', + }); + expect(session.requests[0].compactions).toBeUndefined(); + expect(session.requests[1].compaction).toBeNull(); + expect(session.requests[2].compaction).toBeNull(); + expect(SessionSchema.safeParse(session).success).toBe(true); + }); + + it('keeps every compaction of a turn, and gives one logged before the first prompt to the first request', () => { + const session = parseLines([ + boundary('2026-11-01T09:59:00Z', { trigger: 'manual', preTokens: 90_000 }), + user('long task', '2026-11-01T10:00:00Z'), + assistant('m1', '2026-11-01T10:00:05Z', 200_000), + boundary('2026-11-01T10:00:06Z', { trigger: 'auto', preTokens: 200_000, postTokens: 9_000 }), + assistant('m2', '2026-11-01T10:00:09Z', 9_000), + ]); + const all = requestCompactions(session.requests[0]); + expect(all.map(c => [c.trigger, c.contextLengthBefore, c.contextLengthAfter])).toEqual([ + ['manual', 90_000, undefined], + ['auto', 200_000, 9_000], + ]); + }); +}); diff --git a/src/core/parser-claude.ts b/src/core/parser-claude.ts index 5281138d..db679efd 100644 --- a/src/core/parser-claude.ts +++ b/src/core/parser-claude.ts @@ -18,7 +18,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { createHash } from 'crypto'; -import { Session, SessionRequest } from './types'; +import { CompactionEvent, Session, SessionRequest } from './types'; import { assertTrustedPath, readFileSafe, createRequest, createSession, detectDevcontainerFromRequests, extractSkillNameFromPath } from './parser-shared'; import { extractReasoningEffortFromModelId } from './helpers'; import { warnCore } from './log'; @@ -36,6 +36,8 @@ import { imageReferencesFromContent, type ImageReference, type ImageRequestMatch import { ClaudeResponseAccumulator, type ClaudeRequestUsage } from './parser-claude-responses'; import type { SessionSource } from './cache'; import { forEachJsonlLine } from './parser-vscode-files'; +import { readClaudeCompaction, requestCompactionFields } from './agent-compaction'; +import { addTurnCompaction } from './cli-compaction'; interface ClaudeContentBlock { type: string; @@ -85,6 +87,10 @@ interface ClaudeLine { /** Subagent identifier on `subagents/` files. Present alongside * `isSidechain: true`. Not used for billing math; surfaced for diagnostics. */ agentId?: string; + /** `system` lines: `compact_boundary` marks a compaction. */ + subtype?: unknown; + /** On a `compact_boundary` line: `trigger`, `preTokens`, `postTokens`, `durationMs`. */ + compactMetadata?: unknown; message?: ClaudeMessage; } @@ -120,6 +126,20 @@ interface ClaudeAssistantData { answerEvidence: 'final' | 'legacy' | 'missing'; investigationDelegations: ('research' | 'explore')[]; webDomains: string[]; + /** Compactions logged during the request (`compact_boundary` lines), in order. */ + compactions: CompactionEvent[]; +} + +/** A `compact_boundary` line's compaction, or `null` for any other line. */ +function claudeCompaction(line: ClaudeLine): CompactionEvent | null { + return line.type === 'system' && line.subtype === 'compact_boundary' ? readClaudeCompaction(line.compactMetadata) : null; +} + +/** Put compactions logged before the request's prompt ahead of its own, within the cap. */ +function prependCompactions(data: ClaudeAssistantData, earlier: CompactionEvent[]): void { + if (earlier.length === 0) return; + const own = data.compactions.splice(0); + for (const compaction of [...earlier, ...own]) addTurnCompaction(data.compactions, compaction); } /** Session-wide prompt-cache state across Claude requests. */ @@ -415,6 +435,7 @@ function collectClaudeAssistantData( answerEvidence: 'missing', investigationDelegations: [], webDomains: [], + compactions: [], }; const toolResults = collectClaudeToolResults(lines, startIndex); @@ -422,6 +443,8 @@ function collectClaudeAssistantData( while (i < lines.length) { const next = lines[i]; if (next.type === 'user' && userHasText(next)) break; + const compaction = claudeCompaction(next); + if (compaction) addTurnCompaction(data.compactions, compaction); if (next.type === 'assistant') { const blocks = toContentArray(next.message?.content); const hasText = blocks.some(block => block.type === 'text' && block.text?.trim()); @@ -808,6 +831,7 @@ function buildClaudeRequest( ...(usage.promptCache ? { promptCache: usage.promptCache } : {}), ...(usage.callRowsWithoutCacheData ? { callRowsWithoutCacheData: usage.callRowsWithoutCacheData } : {}), ...(usage.contextUsage ? { contextUsage: usage.contextUsage } : {}), + ...requestCompactionFields(assistantData.compactions), }); mergeRequestEditLoc(editLocIndex, requestId, assistantData.editLocs); return request; @@ -841,11 +865,15 @@ export function parseClaudeSessionFile( let firstTs: number | null = null; let lastTs: number | null = null; let entrypoint: string | undefined; + // Compactions logged before the first prompt go to the first request. + const earlyCompactions: CompactionEvent[] = []; let i = 0; while (i < lines.length) { const line = lines[i]; if (line.type !== 'user' || !userHasText(line)) { + const compaction = requests.length === 0 ? claudeCompaction(line) : null; + if (compaction) addTurnCompaction(earlyCompactions, compaction); i++; continue; } @@ -858,6 +886,7 @@ export function parseClaudeSessionFile( ({ firstTs, lastTs } = updateClaudeTimestampRange(userTs, firstTs, lastTs)); const assistantData = collectClaudeAssistantData(lines, i, lastTs, cacheState); + if (requests.length === 0) prependCompactions(assistantData, earlyCompactions); lastTs = assistantData.lastTs; const requestId = claudeRequestId(line, sessionId, requests.length, transcriptKey); const request = buildClaudeRequest(line, assistantData, userTs, requestId, editLocIndex); diff --git a/src/core/parser-codex-compaction.test.ts b/src/core/parser-codex-compaction.test.ts new file mode 100644 index 00000000..6bfff866 --- /dev/null +++ b/src/core/parser-codex-compaction.test.ts @@ -0,0 +1,98 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/* Codex compactions. Codex logs no sizes for a compaction, and often several records for one: a `compacted` line, + * a `compaction` response item, a `context_compacted` event or a `ContextCompaction` item. The fixtures are + * synthetic, in the shape of Codex rollout files. */ + +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { describe, expect, it } from 'vitest'; +import { parseCodexSessions } from './parser-codex'; +import { requestCompactions } from './context-window'; +import { SessionSchema } from './schemas'; +import type { Session } from './types'; + +function parseLines(lines: object[]): Session { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-compaction-test-')); + const dayDir = path.join(root, 'sessions', '2026', '11', '01'); + fs.mkdirSync(dayDir, { recursive: true }); + fs.writeFileSync(path.join(dayDir, 'rollout-2026-11-01-test.jsonl'), lines.map(l => JSON.stringify(l)).join('\n'), 'utf-8'); + try { + return parseCodexSessions(path.join(root, 'sessions'))[0]; + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +} + +let total = 0; +function tokenCount(ts: string, lastInput: number, window = 258_000): object { + total += lastInput + 10; + return { + type: 'event_msg', timestamp: ts, + payload: { type: 'token_count', info: { + total_token_usage: { input_tokens: total, output_tokens: 10, total_tokens: total }, + last_token_usage: { input_tokens: lastInput, output_tokens: 10 }, + model_context_window: window, + } }, + }; +} + +function prompt(ts: string, message: string): object { + return { type: 'event_msg', timestamp: ts, payload: { type: 'user_message', message } }; +} + +function answer(ts: string): object { + return { type: 'event_msg', timestamp: ts, payload: { type: 'assistant_message', content: 'done' } }; +} + +const HEADER = [ + { type: 'session_meta', payload: { id: 'sess-codex-compact', cwd: '/Users/me/proj' } }, + { type: 'turn_context', payload: { model: 'gpt-5.3-codex' } }, +]; + +describe('Codex compaction', () => { + it('records one compaction per compaction, with the last prompt and window before it', () => { + total = 0; + const session = parseLines([ + ...HEADER, + prompt('2026-11-01T10:00:00Z', 'big task'), + tokenCount('2026-11-01T10:00:05Z', 240_000), + { type: 'compacted', timestamp: '2026-11-01T10:00:06Z', payload: { message: '' } }, + tokenCount('2026-11-01T10:00:06Z', 0), + { type: 'event_msg', timestamp: '2026-11-01T10:00:06Z', payload: { type: 'item_completed', item: { type: 'ContextCompaction', id: 'item-1' } } }, + { type: 'event_msg', timestamp: '2026-11-01T10:00:06Z', payload: { type: 'context_compacted' } }, + tokenCount('2026-11-01T10:00:10Z', 20_000), + answer('2026-11-01T10:00:20Z'), + prompt('2026-11-01T10:01:00Z', 'next task'), + tokenCount('2026-11-01T10:01:05Z', 25_000), + { type: 'response_item', timestamp: '2026-11-01T10:01:06Z', payload: { type: 'compaction', encrypted_content: 'opaque' } }, + tokenCount('2026-11-01T10:01:10Z', 9_000), + answer('2026-11-01T10:01:20Z'), + ]); + expect(session.requests).toHaveLength(2); + expect(requestCompactions(session.requests[0])).toEqual([ + expect.objectContaining({ contextLengthBefore: 240_000, tokenLimit: 258_000, outcome: 'success' }), + ]); + expect(requestCompactions(session.requests[1])).toEqual([ + expect.objectContaining({ contextLengthBefore: 25_000, tokenLimit: 258_000 }), + ]); + expect(SessionSchema.safeParse(session).success).toBe(true); + }); + + it('ignores a compaction a forked thread inherited before its first model call', () => { + total = 0; + const session = parseLines([ + ...HEADER, + { type: 'response_item', timestamp: '2026-11-01T10:00:00Z', payload: { type: 'compaction', encrypted_content: 'opaque' } }, + prompt('2026-11-01T10:00:01Z', 'review this'), + tokenCount('2026-11-01T10:00:05Z', 30_000), + answer('2026-11-01T10:00:20Z'), + ]); + expect(session.requests).toHaveLength(1); + expect(session.requests[0].compaction).toBeNull(); + }); +}); diff --git a/src/core/parser-codex.ts b/src/core/parser-codex.ts index 29605b54..9cfd85dd 100644 --- a/src/core/parser-codex.ts +++ b/src/core/parser-codex.ts @@ -15,11 +15,15 @@ * type=event_msg: payload.type = user_message|agent_reasoning|function_call|function_call_output| * task_started|task_complete|token_count|turn_aborted|error|assistant_message * type=response_item: payload.role, payload.content[] + * type=compacted: a compaction (also a `compaction` response item, a `context_compacted` event or a + * `ContextCompaction` item; see `agent-compaction.ts`) */ import * as fs from 'fs'; import * as path from 'path'; -import { ModelUsage, Session, SessionRequest } from './types'; +import { CompactionEvent, ModelUsage, Session, SessionRequest } from './types'; +import { codexCompaction, requestCompactionFields } from './agent-compaction'; +import { addTurnCompaction } from './cli-compaction'; import { assertTrustedPath, createRequest, createSession, detectDevcontainerFromRequests, extractSkillNameFromPath, extractSkillPathsFromText } from './parser-shared'; import { canonicalizeReasoningEffort, extractReasoningEffortFromModelId, isHarnessInjectedContext } from './helpers'; import { EditLocIndex } from './edit-loc-diff'; @@ -109,6 +113,13 @@ interface CodexParseState { pendingToolEdits: Map; /** The latest `total_token_usage` logged `cached_input_tokens`; without it the cached count is a default 0. */ curTotalCachedReported: boolean; + /** Compactions logged during the current turn, in order. */ + turnCompactions: CompactionEvent[]; + /** Prompt tokens and `model_context_window` of the session's latest main-agent call; 0 and `null` before one. */ + lastPromptTokens: number; + lastContextWindow: number | null; + /** A compaction was recorded since the latest model call: Codex logs one compaction in several records. */ + compactionOpen: boolean; } interface PendingCodexEdit { @@ -205,6 +216,10 @@ function createCodexState(initialModel: string, editLocIndex?: EditLocIndex): Co editLocIndex, pendingToolEdits: new Map(), curTotalCachedReported: false, + turnCompactions: [], + lastPromptTokens: 0, + lastContextWindow: null, + compactionOpen: false, }; } @@ -313,6 +328,7 @@ function flushCodexTurn(state: CodexParseState, defaultModel: string): void { completionTokens: reqCompletionTokens, reasoningEffort: state.turnEffort ?? extractReasoningEffortFromModelId(state.turnModel || defaultModel), ...(contextUsage ? { contextUsage } : {}), + ...requestCompactionFields(state.turnCompactions), })); mergeRequestEditLoc(state.editLocIndex, requestId, state.currentEditLocs); @@ -335,6 +351,7 @@ function resetCodexTurn(state: CodexParseState): void { state.turnStartTs = null; state.turnCanceled = false; state.turnContext = new TurnContextAccumulator(); + state.turnCompactions = []; state.currentEditLocs = new Map(); state.answerEvidence = 'missing'; if (state.imageScan) { @@ -545,10 +562,29 @@ function handleTokenCountEvent(payload: Record, state: CodexPar const last = recordValue(info?.last_token_usage); const lastInput = tokenCount(last?.input_tokens); if (newCall && lastInput != null) { - state.turnContext.addCall({ stream: 'main', inputTokens: lastInput, maxPromptTokens: tokenCount(info?.model_context_window), model: state.turnModel || undefined }); + const window = tokenCount(info?.model_context_window); + state.turnContext.addCall({ stream: 'main', inputTokens: lastInput, maxPromptTokens: window, model: state.turnModel || undefined }); + // Codex logs an empty call (0 input) right after a compaction; only a real call closes it. + if (lastInput > 0) { + state.lastPromptTokens = lastInput; + state.lastContextWindow = window; + state.compactionOpen = false; + } } } +function isCompactionEvent(eventType: string, payload: Record): boolean { + return eventType === 'context_compacted' || (eventType === 'item_completed' && recordValue(payload.item)?.type === 'ContextCompaction'); +} + +/** Record a compaction once, however many records Codex logs for it. One before any model call of the file is + * history a forked thread inherited, not a compaction of this session. */ +function recordCodexCompaction(state: CodexParseState): void { + if (state.compactionOpen || state.lastPromptTokens <= 0) return; + state.compactionOpen = true; + addTurnCompaction(state.turnCompactions, codexCompaction(state.lastPromptTokens, state.lastContextWindow)); +} + function handleEventMsg(payload: Record, state: CodexParseState, ts: number | null, defaultModel: string): void { const eventType = stringValue(payload.type); if (eventType === 'user_message') { @@ -571,6 +607,10 @@ function handleEventMsg(payload: Record, state: CodexParseState handleTokenCount(payload, state); return; } + if (isCompactionEvent(eventType, payload)) { + recordCodexCompaction(state); + return; + } if (eventType === 'function_call_output' || eventType === 'custom_tool_call_output') { handleCodexToolOutput(payload, state); return; @@ -649,7 +689,8 @@ function handleResponseItem(payload: Record, state: CodexParseS handleAssistantResponseItem(payload, state); return; } - if (itemType === 'function_call') handleFunctionCallResponseItem(payload, state); + if (itemType === 'compaction' || itemType === 'compaction_summary') recordCodexCompaction(state); + else if (itemType === 'function_call') handleFunctionCallResponseItem(payload, state); else if (itemType === 'custom_tool_call') handleCustomToolCallResponseItem(payload, state); else if (itemType === 'function_call_output' || itemType === 'custom_tool_call_output') { handleCodexToolOutput(payload, state); @@ -687,6 +728,10 @@ function handleCodexLine(line: CodexLine, state: CodexParseState, meta: CodexSes handleTurnContext(line.payload || {}, state); return; } + if (line.type === 'compacted') { + recordCodexCompaction(state); + return; + } if (line.type === 'response_item') handleResponseItem(line.payload || {}, state, ts, meta.model); }